Apache Kafka można z łatwością uruchomić na swoim komputerze korzystając z gotowych obrazów Dockera. Poniżej kilka przykładów jak to zrobić.
Użyjemy obrazów:
Jeden broker i ZooKeeper
Przygotowanie sieci:
docker network rm bdp_kafka_network
docker network create --driver bridge bdp_kafka_network
docker network ls
Kontener ZooKeepera:
docker run --name zookeeper.bigdatapassion.pl \
-p 2181:2181 \
--network=bdp_kafka_network \
--hostname=zookeeper.bigdatapassion.pl \
-d \
wurstmeister/zookeeper:latest
Kontener Kafki:
docker run --name kafka.bigdatapassion.pl \
-p 9092:9092 \
--env KAFKA_ZOOKEEPER_CONNECT=zookeeper.bigdatapassion.pl \
--network=bdp_kafka_network \
--hostname=kafka.bigdatapassion.pl \
-d \
wurstmeister/kafka:1.0.1
Dodajemy w pliku /etc/hosts:
127.0.0.1 zookeeper.bigdatapassion.pl kafka.bigdatapassion.pl
Żeby zatrzymać i wyczyścić:
docker stop zookeeper.bigdatapassion.pl
docker stop kafka.bigdatapassion.pl
docker rm zookeeper.bigdatapassion.pl
docker rm kafka.bigdatapassion.pl
docker ps -a
Trzech brokerów i ZooKeeper
Jeśli chcemy uruchomić lokalnie Apache Kafka na więcej niż jednym brokerze za pomocą Dockera może użyć do tego Docker Compose
Kafka 1.0.0
version: '3.5'
services:
zookeeper:
container_name: cluster_zookeeper
image: wurstmeister/zookeeper
ports:
- "2181:2181"
networks:
- kafkanetwork
kafka1:
container_name: cluster_kafka1
image: wurstmeister/kafka:1.0.0
ports:
- "9092:9092"
hostname: cluster_kafka1
networks:
- kafkanetwork
environment:
KAFKA_ADVERTISED_LISTENERS: cluster_kafka1
KAFKA_ZOOKEEPER_CONNECT: cluster_zookeeper:2181
depends_on:
- zookeeper
kafka2:
container_name: cluster_kafka2
image: wurstmeister/kafka:1.0.0
ports:
- "9093:9092"
hostname: cluster_kafka2
networks:
- kafkanetwork
environment:
KAFKA_ADVERTISED_LISTENERS: cluster_kafka2
KAFKA_ZOOKEEPER_CONNECT: cluster_zookeeper:2181
depends_on:
- kafka1
kafka3:
container_name: cluster_kafka3
image: wurstmeister/kafka:1.0.0
ports:
- "9094:9092"
hostname: cluster_kafka3
networks:
- kafkanetwork
environment:
KAFKA_ADVERTISED_LISTENERS: cluster_kafka3
KAFKA_ZOOKEEPER_CONNECT: cluster_zookeeper:2181
depends_on:
- kafka2
networks:
kafkanetwork:
driver: bridge
Kafka 1.1.0
version: '3.5'
services:
zookeeper:
container_name: cluster_zookeeper
image: wurstmeister/zookeeper
ports:
- "2181:2181"
networks:
- kafkanetwork
kafka1:
container_name: cluster_kafka1
image: wurstmeister/kafka:1.1.0
ports:
- "9092:9092"
hostname: cluster_kafka1
networks:
- kafkanetwork
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092
KAFKA_ZOOKEEPER_CONNECT: cluster_zookeeper:2181
depends_on:
- zookeeper
kafka2:
container_name: cluster_kafka2
image: wurstmeister/kafka:1.1.0
ports:
- "9093:9092"
hostname: cluster_kafka2
networks:
- kafkanetwork
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092
KAFKA_ZOOKEEPER_CONNECT: cluster_zookeeper:2181
depends_on:
- kafka1
kafka3:
container_name: cluster_kafka3
image: wurstmeister/kafka:1.1.0
ports:
- "9094:9092"
hostname: cluster_kafka3
networks:
- kafkanetwork
environment:
KAFKA_LISTENERS: PLAINTEXT://:9092
KAFKA_ZOOKEEPER_CONNECT: cluster_zookeeper:2181
depends_on:
- kafka2
networks:
kafkanetwork:
driver: bridge
