Docker部署kafka集群

news2024/11/14 13:35:04

1，编写Docker Compose文件
编写一个docker-compose.yml文件来定义服务，以下定义了一个Zookeeper服务和三个Kafka Broker服务：

注意：把10.0.8.4替换成宿主IP

version: '3.8'

services:
  zookeeper:
    image: 'bitnami/zookeeper:latest'
    container_name: ncc-zookeeper
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    networks:
      - kafka-net

  kafka1:
    image: 'bitnami/kafka:latest'
    container_name: ncc-kafka1
    ports:
      - '9092:9092'
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_ADVERTISED_HOST_NAME=10.0.8.4
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_LISTENERS=PLAINTEXT://:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.0.8.4:9092
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    depends_on:
      - zookeeper
    networks:
      - kafka-net

  kafka2:
    image: 'bitnami/kafka:latest'
    container_name: ncc-kafka2
    ports:
      - '9093:9092'
    environment:
      - KAFKA_BROKER_ID=2
      - KAFKA_ADVERTISED_HOST_NAME=10.0.8.4
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_LISTENERS=PLAINTEXT://:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.0.8.4:9093
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    depends_on:
      - zookeeper
    networks:
      - kafka-net

  kafka3:
    image: 'bitnami/kafka:latest'
    container_name: ncc-kafka3
    ports:
      - '9094:9092'
    environment:
      - KAFKA_BROKER_ID=3
      - KAFKA_ADVERTISED_HOST_NAME=10.0.8.4
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_LISTENERS=PLAINTEXT://:9092
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://10.0.8.4:9094
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
    depends_on:
      - zookeeper
    networks:
      - kafka-net

networks:
  kafka-net:
    driver: bridge
    external: false

2，启动Kafka集群

sudo docker-compose up -d

验证安装

一旦所有服务都已启动并运行，你可以通过连接到任意一个Kafka Broker来验证安装是否成功。例如，可以使用Kafka自带的命令行工具创建一个测试主题：

3，在ncc-kafka1创建主题，然后在ncc-kafka2查看主题列表

sudo docker exec -it ncc-kafka1 /opt/bitnami/kafka/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server ncc-kafka1:9092 --replication-factor 3 --partitions 1

sudo docker exec -it ncc-kafka2 /opt/bitnami/kafka/bin/kafka-topics.sh --list --bootstrap-server ncc-kafka2:9092

4，验证发送消息

sudo docker exec -it ncc-kafka1 /opt/bitnami/kafka/bin/kafka-console-producer.sh --broker-list ncc-kafka1:9092 --topic test-topic

另外新开一个窗口，执行

sudo docker exec -it ncc-kafka3 /opt/bitnami/kafka/bin/kafka-console-consumer.sh --bootstrap-server ncc-kafka3:9092 --topic test-topic --from-beginning

测试验证成功