Skip to content

Commit 49ef85c

Browse files
dndanoffdndanoff
committed
add docker-compose for kafka
1 parent 9274612 commit 49ef85c

File tree

6 files changed

+449
-2
lines changed

6 files changed

+449
-2
lines changed

‎DockerCheatSheet.md

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Docker useful commands
2+
3+
## Docker CLI
4+
5+
- Build image from current directory (.) <br>
6+
`docker build --file=Dockerfile --tag=spring-boot-docker-demo:latest --rm=true .`
7+
8+
- Start docker image <br>
9+
`docker run -it --name app -p 8080:8080 -d spring-boot-docker-demo:latest /bin/sh`
10+
11+
- Connect to container OS - use exit to detach <br>
12+
`docker exec -ti app sh`
13+
14+
- Stop container <br>
15+
`docker stop CONTAINER `
16+
17+
- Stop all running containers <br>
18+
`docker stop $(docker ps -aq)`
19+
20+
- Remove container <br>
21+
`docker rm CONTAINER`
22+
23+
- List all containers(even stopped) <br>
24+
`docker container ls --all`
25+
`docker ps -aq`
26+
27+
- Inspect container logs <br>
28+
`docker logs CONTAINER`
29+
30+
- Tail container logs <br>
31+
`docker logs CONTAINER -f`
32+
33+
- List all volumes <br>
34+
`docker volume ls`
35+
36+
- Check volume <br>
37+
`docker volume inspect VOLUME`
38+
39+
- Inspect container <br>
40+
`docker inspect CONTAINER`
41+
42+
- Check container stats <br>
43+
`docker stats`
44+
45+
## Docker Compose
46+
47+
- Start docker <br>
48+
`docker-compose up -d`
49+
50+
- Docker compose remove all containers <br>
51+
`docker-compose rm -f -s -v`

‎README.md

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,48 @@
1-
# kafka-on-docker
2-
Repository to keep different docker-compose setups for Kafka
1+
# Usefull commands when starting Kafka on Docker
2+
3+
## Run with Docker Compose
4+
5+
Move to any of the folders containing docker-compose.yml file.<br>
6+
Folders bring up different stacks. They start from zookeeper, kafka and schema registry and are enhanced to zookeeper, kafka, schema registry, rest proxy, kafka connect, topic-ui, connect-ui, scheam-ui.
7+
8+
- Start kafka platform: <br>
9+
`docker-compose up -d`
10+
11+
- Optional create and check topic: <br>
12+
`docker-compose exec kafka kafka-topics --create --topic foo --partitions 1 --replication-factor 1 --if-not-exists --zookeeper zookeper:2181` <br>
13+
`docker-compose exec kafka kafka-topics --describe --topic foo --zookeeper zookeper:2181` <br>
14+
`docker-compose exec kafka bash -c "seq 42 | kafka-console-producer --request-required-acks 1 --broker-list kafka:9092 --topic foo && echo 'Produced 42 messages.'"` <br>
15+
`docker-compose exec kafka kafka-console-consumer --bootstrap-server kafka:9092 --topic foo --new-consumer --from-beginning --max-messages 42` <br>
16+
17+
- Stop platform: <br>
18+
`docker-compose rm -s -f -v`
19+
20+
## Run with Docker CLI
21+
22+
Run docker for windows with linux containers <br>
23+
https://docs.confluent.io/current/installation/docker/docs/installation/single-node-client.html <br>
24+
25+
- Create network: <br>
26+
`docker network create confluent` <br>
27+
28+
- Start zookeeper: <br>
29+
`docker run -d --net=confluent --name=zookeeper -e ZOOKEEPER_CLIENT_PORT=2181 confluentinc/cp-zookeeper:5.0.0` <br>
30+
31+
- Check zookeeper: <br>
32+
`docker logs zookeeper` <br>
33+
34+
- Start kafka: <br>
35+
`docker run -d --net=confluent --name=kafka -e KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 confluentinc/cp-kafka:5.0.0` <br>
36+
37+
- Check kafka: <br>
38+
`docker logs kafka` <br>
39+
40+
- Start schema registry: <br>
41+
`docker run -d --net=confluent --name=schema-registry -e SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181 -e SCHEMA_REGISTRY_HOST_NAME=schema-registry -e SCHEMA_REGISTRY_LISTENERS=http://0.0.0.0:8081 confluentinc/cp-schema-registry:5.0.0` <br>
42+
43+
- Check schema registry: <br>
44+
`docker logs schema-registry` <br>
45+
46+
- Optional create and check topic: <br>
47+
`docker run --net=confluent --rm confluentinc/cp-kafka:5.0.0 kafka-topics --create --topic foo --partitions 1 --replication-factor 1 --if-not-exists --zookeeper zookeeper:2181` <br>
48+
`docker run --net=confluent --rm confluentinc/cp-kafka:5.0.0 kafka-topics --describe --topic foo --zookeeper zookeeper:2181` <br>

‎kafka-registry-proxy-ui-connect/docker-compose.yml

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
version: "3"
2+
services:
3+
4+
zookeeper:
5+
container_name: zookeeper
6+
image: confluentinc/cp-zookeeper:5.0.0
7+
hostname: zookeeper
8+
ports:
9+
- "2181:2181"
10+
environment:
11+
ZOOKEEPER_CLIENT_PORT: 2181
12+
networks:
13+
- confluent
14+
kafka:
15+
container_name: kafka
16+
image: confluentinc/cp-kafka:5.0.0
17+
hostname: kafka
18+
depends_on:
19+
- zookeeper
20+
ports:
21+
- "9092:9092"
22+
- "19092:19092"
23+
environment:
24+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19092
25+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
26+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
27+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
28+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
29+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
30+
networks:
31+
- confluent
32+
schema-registry:
33+
container_name: schema-registry
34+
image: confluentinc/cp-schema-registry:5.0.0
35+
hostname: schema-registry
36+
depends_on:
37+
- kafka
38+
- zookeeper
39+
ports:
40+
- "8081:8081"
41+
environment:
42+
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
43+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
44+
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
45+
networks:
46+
- confluent
47+
kafka-rest-proxy:
48+
container_name: kafka-rest-proxy
49+
image: confluentinc/cp-kafka-rest:5.0.0
50+
hostname: kafka-rest-proxy
51+
depends_on:
52+
- zookeeper
53+
- kafka
54+
- schema-registry
55+
ports:
56+
- "8082:8082"
57+
environment:
58+
KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181
59+
KAFKA_REST_LISTENERS: http://kafka-rest-proxy:8082
60+
KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081
61+
KAFKA_REST_HOST_NAME: kafka-rest-proxy
62+
networks:
63+
- confluent
64+
kafka-connect:
65+
container_name: kafka-connect
66+
image: confluentinc/cp-kafka-connect:5.0.0
67+
hostname: kafka-connect
68+
ports:
69+
- "8083:8083"
70+
environment:
71+
CONNECT_PLUGIN_PATH: "/usr/share/java,/etc/kafka-connect/jars"
72+
CONNECT_BOOTSTRAP_SERVERS: "kafka:9092"
73+
CONNECT_REST_PORT: 8083
74+
CONNECT_GROUP_ID: "kafka-connect-consumer-group"
75+
CONNECT_CONFIG_STORAGE_TOPIC: "kafka-connect-configs-topic"
76+
CONNECT_OFFSET_STORAGE_TOPIC: "kafka-connect-offsets-topic"
77+
CONNECT_STATUS_STORAGE_TOPIC: "kafka-connect-status-topic"
78+
CONNECT_KEY_CONVERTER: "io.confluent.connect.avro.AvroConverter"
79+
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081/"
80+
CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
81+
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081/"
82+
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
83+
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
84+
CONNECT_REST_ADVERTISED_HOST_NAME: "localhost"
85+
CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
86+
CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect.runtime.rest=WARN,org.reflections=ERROR"
87+
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
88+
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
89+
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
90+
volumes:
91+
- ./kafka-connect/jars:/etc/kafka-connect/jars
92+
depends_on:
93+
- zookeeper
94+
- kafka
95+
- schema-registry
96+
- kafka-rest-proxy
97+
networks:
98+
- confluent
99+
schema-registry-ui:
100+
container_name: schema-registry-ui
101+
image: landoop/schema-registry-ui:0.9.4
102+
hostname: kafka-schema-registry-ui
103+
ports:
104+
- "8001:8000"
105+
environment:
106+
SCHEMAREGISTRY_URL: http://schema-registry:8081/
107+
PROXY: "true"
108+
depends_on:
109+
- schema-registry
110+
networks:
111+
- confluent
112+
kafka-topics-ui:
113+
container_name: kafka-topics-ui
114+
image: landoop/kafka-topics-ui:0.9.4
115+
hostname: kafka-topics-ui
116+
ports:
117+
- "8000:8000"
118+
environment:
119+
KAFKA_REST_PROXY_URL: http://kafka-rest-proxy:8082/
120+
PROXY: "true"
121+
depends_on:
122+
- zookeeper
123+
- kafka
124+
- schema-registry
125+
- kafka-rest-proxy
126+
networks:
127+
- confluent
128+
kafka-connect-ui:
129+
container_name: kafka-connect-ui
130+
image: landoop/kafka-connect-ui:0.9.4
131+
hostname: kafka-connect-ui
132+
ports:
133+
- "8003:8000"
134+
environment:
135+
CONNECT_URL: "http://kafka-connect:8083/"
136+
PROXY: "true"
137+
depends_on:
138+
- kafka-connect
139+
networks:
140+
- confluent
141+
networks:
142+
confluent:

‎kafka-registry-proxy-ui/docker-compose.yml

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
version: "3"
2+
services:
3+
4+
zookeeper:
5+
container_name: zookeeper
6+
image: confluentinc/cp-zookeeper:5.0.0
7+
hostname: zookeeper
8+
ports:
9+
- "2181:2181"
10+
environment:
11+
ZOOKEEPER_CLIENT_PORT: 2181
12+
networks:
13+
- confluent
14+
kafka:
15+
container_name: kafka
16+
image: confluentinc/cp-kafka:5.0.0
17+
hostname: kafka
18+
depends_on:
19+
- zookeeper
20+
ports:
21+
- "9092:9092"
22+
- "19092:19092"
23+
environment:
24+
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:19092
25+
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
26+
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
27+
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
28+
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
29+
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
30+
networks:
31+
- confluent
32+
schema-registry:
33+
container_name: schema-registry
34+
image: confluentinc/cp-schema-registry:5.0.0
35+
hostname: schema-registry
36+
depends_on:
37+
- kafka
38+
- zookeeper
39+
ports:
40+
- "8081:8081"
41+
environment:
42+
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
43+
SCHEMA_REGISTRY_HOST_NAME: schema-registry
44+
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
45+
networks:
46+
- confluent
47+
kafka-rest-proxy:
48+
container_name: kafka-rest-proxy
49+
image: confluentinc/cp-kafka-rest:5.0.0
50+
hostname: kafka-rest-proxy
51+
depends_on:
52+
- zookeeper
53+
- kafka
54+
- schema-registry
55+
ports:
56+
- "8082:8082"
57+
environment:
58+
KAFKA_REST_ZOOKEEPER_CONNECT: zookeeper:2181
59+
KAFKA_REST_LISTENERS: http://kafka-rest-proxy:8082
60+
KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081
61+
KAFKA_REST_HOST_NAME: kafka-rest-proxy
62+
networks:
63+
- confluent
64+
schema-registry-ui:
65+
container_name: schema-registry-ui
66+
image: landoop/schema-registry-ui:0.9.4
67+
hostname: kafka-schema-registry-ui
68+
ports:
69+
- "8001:8000"
70+
environment:
71+
SCHEMAREGISTRY_URL: http://schema-registry:8081/
72+
PROXY: "true"
73+
depends_on:
74+
- schema-registry
75+
networks:
76+
- confluent
77+
kafka-topics-ui:
78+
container_name: kafka-topics-ui
79+
image: landoop/kafka-topics-ui:0.9.4
80+
hostname: kafka-topics-ui
81+
ports:
82+
- "8000:8000"
83+
environment:
84+
KAFKA_REST_PROXY_URL: http://kafka-rest-proxy:8082/
85+
PROXY: "true"
86+
depends_on:
87+
- zookeeper
88+
- kafka
89+
- schema-registry
90+
- kafka-rest-proxy
91+
networks:
92+
- confluent
93+
networks:
94+
confluent:

0 commit comments

Comments
 (0)