基于docker compose 搭建kafka集群_docker-compose部署 kafka:2.13-2.8.1

新建docker网络

docker network create docker-net --subnet 172.20.10.0/16

docker network ls

集群规划

新建一个文件夹 在文件夹下面新建文件zk-docker-compose.yml

version: '2'
 
services:
  zook1:
    image: zookeeper:latest
    #restart: always #自动重新启动
    hostname: zook1
    container_name: zook1 #容器名称，方便在rancher中显示有意义的名称
    ports:
    - 2183:2181 #将本容器的zookeeper默认端口号映射出去
    volumes: # 挂载数据卷 前面是宿主机即本机的目录位置，后面是docker的目录
    - "/Users/konsy/Development/volume/zkcluster/zook1/data:/data"
    - "/Users/konsy/Development/volume/zkcluster/zook1/datalog:/datalog"
    - "/Users/konsy/Development/volume/zkcluster/zook1/logs:/logs"
    environment:
        ZOO_MY_ID: 1  #即是zookeeper的节点值，也是kafka的brokerid值
        ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
    networks:
        docker-net:
            ipv4_address: 172.20.10.11
 
  zook2:   
    image: zookeeper:latest
    #restart: always #自动重新启动
    hostname: zook2
    container_name: zook2 #容器名称，方便在rancher中显示有意义的名称
    ports:
    - 2184:2181 #将本容器的zookeeper默认端口号映射出去
    volumes:
    - "/Users/konsy/Development/volume/zkcluster/zook2/data:/data"
    - "/Users/konsy/Development/volume/zkcluster/zook2/datalog:/datalog"
    - "/Users/konsy/Development/volume/zkcluster/zook2/logs:/logs"
    environment:
        ZOO_MY_ID: 2  #即是zookeeper的节点值，也是kafka的brokerid值
        ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
    networks:
        docker-net:
            ipv4_address: 172.20.10.12
            
  zook3:   
    image: zookeeper:latest
    #restart: always #自动重新启动
    hostname: zook3
    container_name: zook3 #容器名称，方便在rancher中显示有意义的名称
    ports:
    - 2185:2181 #将本容器的zookeeper默认端口号映射出去
    volumes:
    - "/Users/konsy/Development/volume/zkcluster/zook3/data:/data"
    - "/Users/konsy/Development/volume/zkcluster/zook3/datalog:/datalog"
    - "/Users/konsy/Development/volume/zkcluster/zook3/logs:/logs"
    environment:
        ZOO_MY_ID: 3  #即是zookeeper的节点值，也是kafka的brokerid值
        ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181
    networks:
        docker-net:
            ipv4_address: 172.20.10.13
networks:
  docker-net:
      name: docker-net

执行脚本部署zookeeper至Docker:

docker-compose -p zookeeper -f zk-docker-compose.yml up -d

注：在当前目录下执行，如果没有安装docker-compose先去安装docker-compose

Kafka集群安装

新建文件kafka-docker-compose.yml

version: '2'
 
services:
  broker1:
    image: docker.io/wurstmeister/kafka
    #restart: always #自动重新启动
    hostname: broker1
    container_name: broker1
    ports:
      - 9091:9091
      - 9191:9191
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: INSIDE://:9091,OUTSIDE://:9191
      SKAFKA_ADVERTISED_LISTENERS: INSIDE://broker1:9091,OUTSIDE://localhost:9191
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
      JMX_PORT: 9999 #开放JMX监控端口，来监测集群数据
    volumes:
      - /Users/konsy/Development/volume/kafka/broker1/wurstmeister/kafka:/wurstmeister/kafka
      - /Users/konsy/Development/volume/kafka/broker1/kafka:/kafka
    external_links:
      - zook1
      - zook2
      - zook3
    networks:
      docker-net:
        ipv4_address: 172.20.10.14
 
  broker2:
    image: docker.io/wurstmeister/kafka
    #restart: always #自动重新启动
    hostname: broker2
    container_name: broker2
    ports:
      - 9092:9092
      - 9192:9192
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9192
      KAFKA_ADVERTISED_LISTENERS: INSIDE://broker2:9092,OUTSIDE://localhost:9192
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
      JMX_PORT: 9999 #开放JMX监控端口，来监测集群数据
    volumes:
      - /Users/konsy/Development/volume/kafka/broker2/wurstmeister/kafka:/wurstmeister/kafka
      - /Users/konsy/Development/volume/kafka/broker2/kafka:/kafka
    external_links:
      - zook1
      - zook2
      - zook3
    networks:
      docker-net:
        ipv4_address: 172.20.10.15
 
  broker3:
    image: docker.io/wurstmeister/kafka
    #restart: always #自动重新启动
    hostname: broker3
    container_name: broker3
    ports:
      - 9093:9093
      - 9193:9193
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9193
      KAFKA_ADVERTISED_LISTENERS: INSIDE://broker3:9093,OUTSIDE://localhost:9193
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181
      ALLOW_PLAINTEXT_LISTENER : 'yes'
      JMX_PORT: 9999 #开放JMX监控端口，来监测集群数据
    volumes:
      - /Users/konsy/Development/volume/kafka/broker3/wurstmeister/kafka:/wurstmeister/kafka
      - /Users/konsy/Development/volume/kafka/broker3/kafka:/kafka
    external_links:
      - zook1
      - zook2
      - zook3
    networks:
      docker-net:
        ipv4_address: 172.20.10.16
networks:
  docker-net:
      name: docker-net

执行脚本部署zookeeper至Docker:

docker-compose -f kafka-docker-compose.yml up -d

安装kafka-manager

新建文件kafka-manager-docker-compose.yml

version: '2'
 
services:
  kafka-manager:
    image: scjtqs/kafka-manager:latest
    restart: always
    hostname: kafka-manager
    container_name: kafka-manager
    ports:
      - 9000:9002
    external_links:  # 连接本compose文件以外的container
      - zook1
      - zook2
      - zook3
      - broker1
      - broker2
      - broker3
    environment:
      ZK_HOSTS: zook1:2181,zook2:2181,zook3:2181
      KAFKA_BROKERS: broker1:9091,broker2:9092,broker3:9093
      APPLICATION_SECRET: letmein
      KM_ARGS: -Djava.net.preferIPv4Stack=true
    networks:
      docker-net:
        ipv4_address: 172.20.10.10
networks:
  docker-net:
    external:
      name: docker-net

执行脚本部署kafka-manager至Docker:

docker-compose -f kafka-manager-docker-compose.yml up -d

配置Cluster

浏览器打开 172.20.10.10:9002

测试kafka

进入kafka容器的命令行：

docker exec -ti broker1 /bin/bash

进入kafka所在目录：

cd opt/kafka_2.13-2.8.1/

创建Replication为2，Partition为2的topic

bin/kafka-topics.sh --create --zookeeper zook1:2181 --replication-factor 2 --partitions 2 --topic partopic

查看topic的状态

在kafka容器中的 opt/kafka_2.12-1.1.0/ 目录下输入

bin/kafka-topics.sh --describe --zookeeper zook1:2181 --topic partopic