基于Docker-compose构建Kafka集群_docker kafka 集群 编排

0.前言

首先，先向大家道个歉。由于前段时间的工作调整，导致我一直没有进行更新。现在工作逐渐走上正轨，现在决定继续更新自己学习的内容。希望对于学习或者从事大数据开发的你提供一些帮助。
本次更新的内容为：如何使用Docker-compose技术进行Kafka 集群的构建

1.Docker-compose编排Kafka单节点

Docker-compose是一种对于Docker容器进行编排的技术。使用Docker-compose技术可以很容易的进行Docker容器的编排。
本次对于Kafka集群的编排，我们采用第三方镜像（bitnami）的镜像进行Kafka集群的编排。其中GitHub地址为：https://github.com/bitnami/bitnami-docker-kafka
下面是官方提供的单节点的Kafka的容器编排的模板文件：

version: "2"

services:
  zookeeper:
    image: docker.io/bitnami/zookeeper:3.8
    ports:
      - "2181:2181"
    volumes:
      - "zookeeper_data:/bitnami"
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
  kafka:
    image: docker.io/bitnami/kafka:3.2
    ports:
      - "9092:9092"
    volumes:
      - "kafka_data:/bitnami"
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper

volumes:
  zookeeper_data:
    driver: local
  kafka_data:
    driver: local
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

2.Docker-compose编排Kafka集群

对于生产环境，我们需要配置Kafka集群，进而提高系统的稳定性和容错性，因此官方提供了一个用Docker-compose进行编排Kafka的集群模板文件：

version: "2"

services:
  zookeeper:
    image: docker.io/bitnami/zookeeper:3.8
    ports:
      - "2181"
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
    volumes:
      - zookeeper_data:/bitnami/zookeeper
  kafka-0:
    image: docker.io/bitnami/kafka:3.2
    ports:
      - "9092"
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_BROKER_ID=0
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - kafka_0_data:/bitnami/kafka
    depends_on:
      - zookeeper
  kafka-1:
    image: docker.io/bitnami/kafka:3.2
    ports:
      - "9092"
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_BROKER_ID=1
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - kafka_1_data:/bitnami/kafka
    depends_on:
      - zookeeper
  kafka-2:
    image: docker.io/bitnami/kafka:3.2
    ports:
      - "9092"
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_CFG_BROKER_ID=2
      - ALLOW_PLAINTEXT_LISTENER=yes
    volumes:
      - kafka_2_data:/bitnami/kafka
    depends_on:
      - zookeeper

volumes:
  zookeeper_data:
    driver: local
  kafka_0_data:
    driver: local
  kafka_1_data:
    driver: local
  kafka_2_data:
    driver: local
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

3.Docker-compose编排内外网通过的Kafka集群

经过测试得出，官方提供的Kafka集群的模板文件，只能实现内部网络通信，如果需要进行外部访问时，需要对于Docker-compose文件进行修改。下面是我通过自定义网络模式，对于Kafka集群进行编排，进而实现内外网访问的Kafka集群：

version: '2'
networks:
  itoe-network:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 192.168.110.0/24
          gateway: 192.168.110.1

services:
  zookeeper-server:
    image: 'bitnami/zookeeper:latest'
    networks:
      itoe-network:
        ipv4_address: 192.168.110.101
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes



  kafka-server1:
    image: 'bitnami/kafka:latest'
    networks:
      itoe-network:
        ipv4_address: 192.168.110.102
    ports:
      - '9093:9093'
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server:2181
      - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka-server1:9092,EXTERNAL://localhost:9093
      - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9093
      - ALLOW_PLAINTEXT_LISTENER=yes

    depends_on:
      - zookeeper-server


  kafka-server2:
    image: 'bitnami/kafka:latest'
    networks:
      itoe-network:
        ipv4_address: 192.168.110.103
    ports:
      - '9094:9094'
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server:2181
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT
      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka-server2:9092,EXTERNAL://localhost:9094
      - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9094
      - ALLOW_PLAINTEXT_LISTENER=yes

    depends_on:
      - zookeeper-server


  kafka-server3:
    image: 'bitnami/kafka:latest'
    networks:
      itoe-network:
        ipv4_address: 192.168.110.104
    ports:
      - '9095:9095'
    environment:
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper-server:2181
      - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT
      - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT
      - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka-server3:9092,EXTERNAL://localhost:9095
      - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9095
      - ALLOW_PLAINTEXT_LISTENER=yes

    depends_on:
      - zookeeper-server
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79