k8s部署最新版zookeeper集群(3.9.2),并配置prometheus监控

zookeeper集群部署

复制粘贴即用

创建zookeeper文件夹

# 创建文件夹
mkdir -p /opt/zookeeper
cd /opt/zookeeper
1
2
3

namespace.yaml

apiVersion: v1
kind: Namespace
metadata:
  labels:
    kubernetes.io/metadata.name: zookeeper
  name: zookeeper
1
2
3
4
5
6

scripts-configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: zookeeper-scripts
  namespace: zookeeper
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/version: 3.9.2
    app.kubernetes.io/component: zookeeper
data:
  init-certs.sh: |-
    #!/bin/bash
  setup.sh: |-
    #!/bin/bash

    # Execute entrypoint as usual after obtaining ZOO_SERVER_ID
    # check ZOO_SERVER_ID in persistent volume via myid
    # if not present, set based on POD hostname
    if [[ -f "/bitnami/zookeeper/data/myid" ]]; then
        export ZOO_SERVER_ID="$(cat /bitnami/zookeeper/data/myid)"
    else
        HOSTNAME="$(hostname -s)"
        if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
            ORD=${BASH_REMATCH[2]}
            export ZOO_SERVER_ID="$((ORD + 1 ))"
        else
            echo "Failed to get index from hostname $HOSTNAME"
            exit 1
        fi
    fi
    exec /entrypoint.sh /run.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

serviceaccount.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: zookeeper
  namespace: zookeeper
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/version: 3.9.2
    app.kubernetes.io/component: zookeeper
    role: zookeeper
automountServiceAccountToken: false
1
2
3
4
5
6
7
8
9
10
11

statefulset.yaml

---
# Source: zookeeper/templates/statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: zookeeper
  namespace: zookeeper
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/version: 3.9.2
    app.kubernetes.io/component: zookeeper
    role: zookeeper
spec:
  replicas: 3
  revisionHistoryLimit: 10
  podManagementPolicy: Parallel
  selector:
    matchLabels:
      app.kubernetes.io/name: zookeeper
      app.kubernetes.io/component: zookeeper
  serviceName: zookeeper-headless
  updateStrategy:
    rollingUpdate: {}
    type: RollingUpdate
  template:
    metadata:
      annotations:
      labels:
        app.kubernetes.io/name: zookeeper
        app.kubernetes.io/version: 3.9.2
        app.kubernetes.io/component: zookeeper
    spec:
      enableServiceLinks: true
      serviceAccountName: zookeeper
      # 取消为 service account 自动挂载 API 凭证
      automountServiceAccountToken: false
      affinity:
        podAffinity:
        # pod反亲和，将pod打散至不同节点，实现高可用
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - podAffinityTerm:
                labelSelector:
                  matchLabels:
                    app.kubernetes.io/name: zookeeper
                    app.kubernetes.io/component: zookeeper
                topologyKey: kubernetes.io/hostname
              weight: 1
        nodeAffinity:
      # 配置安全上下文    
      securityContext:
        fsGroup: 1001
        fsGroupChangePolicy: Always
        supplementalGroups: []
        sysctls: []
      initContainers:
      containers:
        - name: zookeeper
          image: docker.io/bitnami/zookeeper:3.9.2-debian-12-r2
          imagePullPolicy: "IfNotPresent"
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop:
              - ALL
            privileged: false
            readOnlyRootFilesystem: true
            runAsGroup: 1001
            runAsNonRoot: true
            runAsUser: 1001
            seLinuxOptions: {}
            seccompProfile:
              type: RuntimeDefault
          command:
            - /scripts/setup.sh
          resources:
            limits:
              cpu: 375m
              ephemeral-storage: 1024Mi
              memory: 384Mi
            requests:
              cpu: 250m
              ephemeral-storage: 50Mi
              memory: 256Mi
          env:
            - name: BITNAMI_DEBUG
              value: "false"
            - name: ZOO_DATA_LOG_DIR
              value: ""
            - name: ZOO_PORT_NUMBER
              value: "2181"
            - name: ZOO_TICK_TIME
              value: "2000"
            - name: ZOO_INIT_LIMIT
              value: "10"
            - name: ZOO_SYNC_LIMIT
              value: "5"
            - name: ZOO_PRE_ALLOC_SIZE
              value: "65536"
            - name: ZOO_SNAPCOUNT
              value: "100000"
            - name: ZOO_MAX_CLIENT_CNXNS
              value: "60"
            - name: ZOO_4LW_COMMANDS_WHITELIST
              value: "srvr, mntr, ruok"
            - name: ZOO_LISTEN_ALLIPS_ENABLED
              value: "no"
            - name: ZOO_AUTOPURGE_INTERVAL
              value: "1"
            - name: ZOO_AUTOPURGE_RETAIN_COUNT
              value: "10"
            - name: ZOO_MAX_SESSION_TIMEOUT
              value: "40000"
            - name: ZOO_SERVERS
              value: zookeeper-0.zookeeper-headless.zookeeper.svc.cluster.local:2888:3888::1 zookeeper-1.zookeeper-headless.zookeeper.svc.cluster.local:2888:3888::2 zookeeper-2.zookeeper-headless.zookeeper.svc.cluster.local:2888:3888::3 
            - name: ZOO_ENABLE_AUTH
              value: "no"
            - name: ZOO_ENABLE_QUORUM_AUTH
              value: "no"
            - name: ZOO_HEAP_SIZE
              value: "1024"
            - name: ZOO_LOG_LEVEL
              value: "ERROR"
            - name: ALLOW_ANONYMOUS_LOGIN
              value: "yes"
            - name: ZOO_ENABLE_PROMETHEUS_METRICS
              value: "yes"
            - name: ZOO_PROMETHEUS_METRICS_PORT_NUMBER
              value: "9141"
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: ZOO_ADMIN_SERVER_PORT_NUMBER
              value: "8080"
          ports:
            - name: client
              containerPort: 2181
            - name: follower
              containerPort: 2888
            - name: election
              containerPort: 3888
            # 开启metrics，供prometheus监控
            - name: metrics
              containerPort: 9141
            - name: http-admin
              containerPort: 8080
          livenessProbe:
            failureThreshold: 6
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            exec:
              command:
                - /bin/bash
                - -ec
                - ZOO_HC_TIMEOUT=2 /opt/bitnami/scripts/zookeeper/healthcheck.sh
          readinessProbe:
            failureThreshold: 6
            initialDelaySeconds: 5
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 5
            exec:
              command:
                - /bin/bash
                - -ec
                - ZOO_HC_TIMEOUT=2 /opt/bitnami/scripts/zookeeper/healthcheck.sh
          volumeMounts:
            - name: empty-dir
              mountPath: /tmp
              subPath: tmp-dir
            - name: empty-dir
              mountPath: /opt/bitnami/zookeeper/conf
              subPath: app-conf-dir
            - name: empty-dir
              mountPath: /opt/bitnami/zookeeper/logs
              subPath: app-logs-dir
            - name: scripts
              mountPath: /scripts/setup.sh
              subPath: setup.sh
            - name: data
              mountPath: /bitnami/zookeeper
      volumes:
        - name: empty-dir
          emptyDir: {}
        - name: scripts
          configMap:
            name: zookeeper-scripts
            defaultMode: 493
  volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes:
          - "ReadWriteOnce"
        resources:
          requests:
            storage: "10Gi"
        # 按当前集群存储类修改    
        storageClassName: nfs-client
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203

svc-headless.yaml

apiVersion: v1
kind: Service
metadata:
  name: zookeeper-headless
  namespace: zookeeper
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/version: 3.9.2
    app.kubernetes.io/component: zookeeper
spec:
  type: ClusterIP
  clusterIP: None
  publishNotReadyAddresses: true
  ports:
    - name: tcp-client
      port: 2181
      targetPort: client
    - name: tcp-follower
      port: 2888
      targetPort: follower
    - name: tcp-election
      port: 3888
      targetPort: election
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: zookeeper
  namespace: zookeeper
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/version: 3.9.2
    app.kubernetes.io/component: zookeeper
spec:
  type: ClusterIP
  sessionAffinity: None
  ports:
    - name: tcp-client
      port: 2181
      targetPort: client
      nodePort: null
    - name: tcp-follower
      port: 2888
      targetPort: follower
    - name: tcp-election
      port: 3888
      targetPort: election
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

metrics-svc.yaml

apiVersion: v1
kind: Service
metadata:
  name: zookeeper-metrics
  namespace: zookeeper
  labels:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/version: 3.9.2
    app.kubernetes.io/component: metrics
  annotations:
    prometheus.io/path: /metrics
    prometheus.io/port: "9141"
    # 开启prometheus监控
    prometheus.io/scrape: "true"
spec:
  type: ClusterIP
  ports:
    - name: tcp-metrics
      port: 9141
      targetPort: metrics
  selector:
    app.kubernetes.io/name: zookeeper
    app.kubernetes.io/component: zookeeper
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

执行部署

# 执行部署
kubectl apply -f .
# 查看部署情况
kubectl get pod,svc,pvc -n zookeeper
NAME              READY   STATUS    RESTARTS   AGE
pod/zookeeper-0   1/1     Running   0          16h
pod/zookeeper-1   1/1     Running   0          16h
pod/zookeeper-2   1/1     Running   0          16h

NAME                         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                      AGE
service/zookeeper            ClusterIP   10.0.6.237   <none>        2181/TCP,2888/TCP,3888/TCP   19h
service/zookeeper-headless   ClusterIP   None         <none>        2181/TCP,2888/TCP,3888/TCP   19h
service/zookeeper-metrics    ClusterIP   10.0.8.208   <none>        9141/TCP                     19h

NAME                                     STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
persistentvolumeclaim/data-zookeeper-0   Bound    pvc-a6ac5f14-bd56-4d20-8e8b-64b8efe4ab52   10Gi       RWO            nfs-client     19h
persistentvolumeclaim/data-zookeeper-1   Bound    pvc-7beccb60-202f-4be4-90e0-2178385055fb   10Gi       RWO            nfs-client     19h
persistentvolumeclaim/data-zookeeper-2   Bound    pvc-7c5ed75d-0467-4007-b5f1-7a7701b91b92   10Gi       RWO            nfs-client     19h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

接入prometheus

prometheus部署可查看历史博客:k8s-prometheus+grafana+alertmanager监控加邮件告警

访问prometheus查看接入情况

导入zookeeper监控模版

模版编号: 10465

监控展示

注：刚部署完监控值皆为0，图中为接入业务后有数据显示