docker-compose 搭建kafka集群_docker-compose 部署kafka集群

docker-compose搭建kafka集群

下载镜像

1.wurstmeister/zookeeper
2.wurstmeister/kafka
3.sheepkiller/kafka-manager
1
2
3

安装docker-compose(如果没有)

https://docs.docker.com/compose/install/
1

在每台机器创建文件夹(节点)

mkdir -p /database/kafka_log/zookeeper/zk1/conf
mkdir -p /database/kafka_log/zookeeper/zk1/data 
mkdir -p /database/kafka_log/zookeeper/zk1/log 
mkdir -p /database/kafka_log/zookeeper/zk2/conf 
mkdir -p /database/kafka_log/zookeeper/zk2/data 
mkdir -p /database/kafka_log/zookeeper/zk2/log 
mkdir -p /database/kafka_log/zookeeper/zk3/conf 
mkdir -p /database/kafka_log/zookeeper/zk3/data 
mkdir -p /database/kafka_log/zookeeper/zk3/log 
mkdir -p /database/kafka_log/kafka/kafka1/conf 
mkdir -p /database/kafka_log/kafka/kafka1/log 
mkdir -p /database/kafka_log/kafka/kafka2/conf 
mkdir -p /database/kafka_log/kafka/kafka2/log 
mkdir -p /database/kafka_log/kafka/kafka3/conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14

创建docker网桥

1.docker network list 查看已生成的网桥
2.docker network inspect *****  (*****:网桥的名称)
3.docker  network  create  --driver  bridge  --subnet  172.16.0.0/25  --gateway  172.16.0.1  zookeeper_kafka 生成新的网桥(ip根据自己的docker网桥号的规律创建)
4.(可通信跳过)如果多机器网桥无法通信  ufw allow from 172.19.0.0/16  （172.19.0.0/16：容器的IP 和使用的端口） ufw 若果没有安装即可
1
2
3
4

创建docker-compose文件(节点一)

version: '3.7'
networks:
  zookeeper_kafka:
    external: true
services:
  zookeeper1:
    image: wurstmeister/zookeeper
    restart: unless-stopped
    hostname: zookeeper1
    container_name: zookeeper1
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOO_MY_ID: 10 ## 唯一标识 不可重复 （同一个节点server和kafka保持一致）
      ZOO_SERVERS: server.10=192.168.1.181:2888:3888;2181 server.20=192.168.1.169:2888:3888;2181 server.30=192.168.1.129:2888:3888;2181  ##zookeeper节点信息
    volumes:
      # - /database/kafka_log/zookeeper/zk1/conf/zoo.cfg:/opt/zookeeper-3.4.13/conf/zoo.cfg
      - /database/kafka_log/zookeeper/zk1/data:/opt/zookeeper-3.4.13/data
      - /database/kafka_log/zookeeper/zk1/log:/opt/zookeeper-3.4.13/datalog
    networks:
      - zookeeper_kafka

kafka1:
    image: wurstmeister/kafka
    restart: unless-stopped
    container_name: kafka1
    hostname: kafka1
    ports:
      - "9092:9092"
    external_links:   # 连接本compose文件以外的container 节点ip 或者hostname
      - 192.168.1.181
      - 192.168.1.169
      - 192.168.1.129
    environment:
      KAFKA_BROKER_ID: 10
      KAFKA_ADVERTISED_HOST_NAME: 192.168.1.181                   ## 修改:宿主机IP
      KAFKA_ADVERTISED_PORT: 9092                                 ## 修改:宿主机映射port
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.181:9092    ## 绑定发布订阅的端口。修改:宿主机IP
      KAFKA_ZOOKEEPER_CONNECT: "192.168.1.181:2181,192.168.1.169:2181,192.168.1.129:2181"
    volumes: 挂载本地文件
      - "/var/run/docker.sock:/var/run/docker.sock"
      - "/database/kafka_log/kafka/kafka1/log:/kafka"
    networks:
      - zookeeper_kafka
      
kafka-manager:
    image: sheepkiller/kafka-manager:latest
    restart: unless-stopped
    container_name: kafka-manager
    hostname: kafka-manager
    ports:
      - "9000:9000"
    # links:            # 连接本compose文件创建的container
      # - kafka1
      # - kafka2
      # - kafka3
    external_links:   # 连接本compose文件以外的container 节点ip 或者hostname
      - 192.168.1.181
      - 192.168.1.169
      - 192.168.1.129
    environment:
      ZK_HOSTS: 192.168.1.181:2181,192.168.1.169:2181,192.168.1.129:2181                ## 修改:宿主机IP
      TZ: CST-8
    networks:
      - zookeeper_kafka
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

创建docker-compose文件(节点二)

version: '3.7'
networks:
  zookeeper_kafka:
    external: true
services:
  zookeeper2:
    image: wurstmeister/zookeeper
    restart: unless-stopped
    hostname: zookeeper2
    container_name: zookeeper2
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOO_MY_ID: 20
      ZOO_SERVERS: server.10=192.168.1.181:2888:3888;2181 server.20=192.168.1.169:2888:3888;2181 server.30=192.168.1.129:2888:3888;2181
    volumes:
      # - /database/kafka_log/zookeeper/zk2/conf/zoo.cfg:/opt/zookeeper-3.4.13/conf/zoo.cfg
      - /database/kafka_log/zookeeper/zk2/data:/opt/zookeeper-3.4.13/data
      - /database/kafka_log/zookeeper/zk2/log:/opt/zookeeper-3.4.13/datalog
    networks:
      - zookeeper_kafka

kafka2:
    image: wurstmeister/kafka
    restart: unless-stopped
    container_name: kafka2
    hostname: kafka2
    ports:
      - "9092:9092"
    external_links:
      - 192.168.1.181
      - 192.168.1.169
      - 192.168.1.129
    environment:
      KAFKA_BROKER_ID: 20
      KAFKA_ADVERTISED_HOST_NAME: 192.168.1.169                 ## 修改:宿主机IP
      KAFKA_ADVERTISED_PORT: 9092                               ## 修改:宿主机映射port
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.169:9092   ## 修改:宿主机IP
      KAFKA_ZOOKEEPER_CONNECT: "192.168.1.181:2181,192.168.1.169:2181,192.168.1.129:2181"
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
      - "/database/kafka_log/kafka/kafka2/log:/kafka"
    networks:
      - zookeeper_kafka
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

创建docker-compose文件(节点…)

version: '3.7'

networks:
  zookeeper_kafka:
    external: true

services:
  zookeeper3:
    image: wurstmeister/zookeeper
    restart: unless-stopped
    hostname: zookeeper3
    container_name: zookeeper3
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOO_MY_ID: 30
      ZOO_SERVERS: server.10=192.168.1.181:2888:3888;2181 server.20=192.168.1.169:2888:3888;2181 server.30=192.168.1.129:2888:3888;2181
    volumes:
      # - /database/kafka_log/zookeeper/zk3/conf/zoo.cfg:/opt/zookeeper-3.4.13/conf/zoo.cfg
      - /database/kafka_log/zookeeper/zk3/data:/opt/zookeeper-3.4.13/data
      - /database/kafka_log/zookeeper/zk3/log:/opt/zookeeper-3.4.13/datalog
    networks:
      - zookeeper_kafka

kafka3:
    image: wurstmeister/kafka
    restart: unless-stopped
    container_name: kafka3
    hostname: kafka3
    ports:
      - "9092:9092"
    external_links:
      - 192.168.1.181
      - 192.168.1.169
      - 192.168.1.129
    environment:
      KAFKA_BROKER_ID: 30
      KAFKA_ADVERTISED_HOST_NAME: 192.168.1.129                 ## 修改:宿主机IP
      KAFKA_ADVERTISED_PORT: 9092                              ## 修改:宿主机映射port
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.1.129:9092   ## 修改:宿主机IP
      KAFKA_ZOOKEEPER_CONNECT: "192.168.1.181:2181,192.168.1.169:2181,192.168.1.129:2181"
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
      - "/database/kafka_log/kafka/kafka3/log:/kafka"
    networks:
      - zookeeper_kafka
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

注意

1. docker挂载的目录修改为自己机器创建的目录 
2. 查看镜像中zookeeper 的版本
3. 安装时docker指令 进行了rm 或者 down  删除容器 重新启动 要删除日志目录下的meta.properties文件 
1
2
3