k8s 1.23.1部署gitlab、 gitlab-runn配置k8s集群编写.gitlab-ci.yml实现根据tag,分支发布CICD流程_gitlab-ci.yml怎么根据分支处理

1. k8s环境部署：https://blog.csdn.net/weixin_43606975/article/details/119947061?spm=1001.2014.3001.5501
2. k8s部署gitlab

#cat gitlab-deploy.yaml
apiVersion: v1
kind: Service
metadata:
  name: gitlab
spec:
  type: NodePort
  ports:
  # Port上的映射端口
  - port: 443
    targetPort: 443
    nodePort: 443
    name: gitlab443
  - port: 80
    nodePort: 80
    targetPort: 80
    name: gitlab80
  - port: 22
    targetPort: 22
    name: gitlab22
  selector:
    app: gitlab

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab
spec:
  selector:
    matchLabels:
      app: gitlab
  revisionHistoryLimit: 2
  template:
    metadata:
      labels:
        app: gitlab
    spec:
      containers:
      # 应用的镜像
      - image: gitlab/gitlab-ce
        name: gitlab
        imagePullPolicy: IfNotPresent
        # 应用的内部端口
        ports:
        - containerPort: 443
          name: gitlab443
        - containerPort: 80
          name: gitlab80
        - containerPort: 22
          name: gitlab22
        volumeMounts:
        # gitlab持久化
        - name: gitlab-persistent-config
          mountPath: /etc/gitlab
       # - name: gitlab-persistent-logs
       #   mountPath: /var/log/gitlab
        - name: gitlab-persistent-data
          mountPath: /var/opt/gitlab
      imagePullSecrets:
      - name: devops-repo
      volumes:
      # 使用本地存储，用于测试，当然你可以不用k8s部署，k8s稍微快一点。
      - name: gitlab-persistent-config
        hostPath:
          path: /home/gitlab/config
      - name:  gitlab-persistent-data
        hostPath:
          path: /home/gitlab/data

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

3. 启动访问

kubectl apply -f gitlab-deploy.yaml
#查看密码：在挂载的node的目录上或者进去容器里面
cat initial_root_password
#容器
cat /etc/gitlab/initial_root_password
1
2
3
4
5

4. gitlab配置k8s集群
   
   
   
   Kubernetes cluster name: 随意
   API URL：

kubectl cluster-info
1

CA Certificate：

kubectl get secret $(kubectl get secret | grep default-token | awk '{print $1}') -o jsonpath="{['data']['ca\.crt']}" | base64 --decode
1

Enter new Service Token：
需要创建gitlab认证：

#cat gitlab-admin-service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitlab-admin
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: gitlab-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: gitlab-admin
  namespace: kube-system
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19

kubectl apply -f gitlab-admin-service-account.yaml
kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep gitlab-admin | awk '{print $1}')
1
2

5. 保存可能会报错：

Requests to the local network are not allowed
1

gitlab需要设置一下：

6. 部署gitlab-Runner
   官网：https://docs.gitlab.com/runner/install/kubernetes-agent.html

#cat runner-manifest.yaml
---
# Source: gitlab-runner/templates/service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  annotations:
  name: gitlab-runner
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
---
# Source: gitlab-runner/templates/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: "gitlab-runner"
  namespace: "gitlab"
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
type: Opaque
data:
  runner-registration-token: "UUNHRGY1WVM0QXNpdFA0eDZRQkw="   #echo VhSA7XiDmU65LoHxN-Gs | base64 -w0
  runner-token: ""
---
# Source: gitlab-runner/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: gitlab-runner
  namespace: "gitlab"
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
data:
  entrypoint: |
    #!/bin/bash
    set -e

    mkdir -p /home/gitlab-runner/.gitlab-runner/

    cp /configmaps/config.toml /home/gitlab-runner/.gitlab-runner/

    # Set up environment variables for cache
    if [[ -f /secrets/accesskey && -f /secrets/secretkey ]]; then
      export CACHE_S3_ACCESS_KEY=$(cat /secrets/accesskey)
      export CACHE_S3_SECRET_KEY=$(cat /secrets/secretkey)
    fi

    if [[ -f /secrets/gcs-applicaton-credentials-file ]]; then
      export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-applicaton-credentials-file"
    elif [[ -f /secrets/gcs-application-credentials-file ]]; then
      export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-application-credentials-file"
    else
      if [[ -f /secrets/gcs-access-id && -f /secrets/gcs-private-key ]]; then
        export CACHE_GCS_ACCESS_ID=$(cat /secrets/gcs-access-id)
        # echo -e used to make private key multiline (in google json auth key private key is oneline with \n)
        export CACHE_GCS_PRIVATE_KEY=$(echo -e $(cat /secrets/gcs-private-key))
      fi
    fi

    if [[ -f /secrets/azure-account-name && -f /secrets/azure-account-key ]]; then
      export CACHE_AZURE_ACCOUNT_NAME=$(cat /secrets/azure-account-name)
      export CACHE_AZURE_ACCOUNT_KEY=$(cat /secrets/azure-account-key)
    fi

    if [[ -f /secrets/runner-registration-token ]]; then
      export REGISTRATION_TOKEN=$(cat /secrets/runner-registration-token)
    fi

    if [[ -f /secrets/runner-token ]]; then
      export CI_SERVER_TOKEN=$(cat /secrets/runner-token)
    fi

    # Validate this also at runtime in case the user has set a custom secret
    if [[ ! -z "$CI_SERVER_TOKEN" && "1" -ne "1" ]]; then
      echo "Using a runner token with more than 1 replica is not supported."
      exit 1
    fi

    # Register the runner
    if ! sh /configmaps/register-the-runner; then
      exit 1
    fi

    # Run pre-entrypoint-script
    if ! bash /configmaps/pre-entrypoint-script; then
      exit 1
    fi

    # Start the runner
    exec /entrypoint run --user=gitlab-runner \
      --working-directory=/home/gitlab-runner

  config.toml: |
    concurrent = 10
    check_interval = 30
    log_level = "info"
    listen_address = ':9252'


  config.template.toml:   |
    [[runners]]
      [runners.kubernetes]
        namespace = "gitlab"
        image = "ubuntu:16.04"


  register-the-runner: |
    #!/bin/bash
    MAX_REGISTER_ATTEMPTS=30

    for i in $(seq 1 "${MAX_REGISTER_ATTEMPTS}"); do
      echo "Registration attempt ${i} of ${MAX_REGISTER_ATTEMPTS}"
      /entrypoint register \
        --run-untagged=true \
        --template-config /configmaps/config.template.toml \
        --non-interactive

      retval=$?

      if [ ${retval} = 0 ]; then
        break
      elif [ ${i} = ${MAX_REGISTER_ATTEMPTS} ]; then
        exit 1
      fi

      sleep 5
    done

    exit 0

  check-live: |
    #!/bin/bash
    if /usr/bin/pgrep -f .*register-the-runner; then
      exit 0
    elif /usr/bin/pgrep gitlab.*runner; then
      exit 0
    else
      exit 1
    fi

  pre-entrypoint-script: |
---
# Source: gitlab-runner/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: "Role"
metadata:
  name: gitlab-runner
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
  namespace: "gitlab"
rules:
- apiGroups: [""]
  resources: ["pods","pods/exec","secrets"]
  verbs: ["get","list","watch","create","patch","delete"]
---
# Source: gitlab-runner/templates/role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: "RoleBinding"
metadata:
  name: gitlab-runner
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
  namespace: "gitlab"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: "Role"
  name: gitlab-runner
subjects:
- kind: ServiceAccount
  name: gitlab-runner
  namespace: "gitlab"
---
# Source: gitlab-runner/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab-runner
  namespace: "gitlab"
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: gitlab-runner
  template:
    metadata:
      labels:
        app: gitlab-runner
        chart: gitlab-runner-0.42.0
        release: "gitlab-runner"
        heritage: "Helm"
      annotations:
        checksum/configmap: 0eb770c0b5b3282a9833fb5ccec0ecf843db4e79ac5a3c6fb696077df5abde21
        checksum/secrets: 0b3f7c87e11d8e0da7056f3ade5bd18756073ff52a72e519c98c0dff5059e9c2
        prometheus.io/scrape: 'true'
        prometheus.io/port: "9252"
    spec:
      securityContext:
        runAsUser: 100
      terminationGracePeriodSeconds: 3600
      serviceAccountName: gitlab-runner
      containers:
      - name: gitlab-runner
        image: registry.gitlab.com/gitlab-org/gitlab-runner:alpine-v15.1.0
        imagePullPolicy: "IfNotPresent"
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          privileged: false
          readOnlyRootFilesystem: false
          runAsNonRoot: true
          runAsUser: 100
        lifecycle:
          preStop:
            exec:
              command: ["/entrypoint", "unregister", "--config=/home/gitlab-runner/.gitlab-runner/config.toml"]
        lifecycle:
          preStop:
            exec:
              command: ["/entrypoint", "unregister", "--all-runners"]
        command: ["/usr/bin/dumb-init", "--", "/bin/bash", "/configmaps/entrypoint"]
        env:

        - name: CI_SERVER_URL
          value: "http://192.168.10.100/"    #gitlab地址
        - name: CLONE_URL
          value: ""
        - name: RUNNER_EXECUTOR
          value: "kubernetes"
        - name: REGISTER_LOCKED
          value: "false"
        - name: RUNNER_TAG_LIST
          value: "k8s-runner,k8s"
        - name: KUBERNETES_PRIVILEGED
          value: "true"
        livenessProbe:
          exec:
            command: ["/bin/bash", "/configmaps/check-live"]
          initialDelaySeconds: 60
          timeoutSeconds: 1
          periodSeconds: 10
          successThreshold: 1
          failureThreshold: 3
        readinessProbe:
          exec:
            command: ["/usr/bin/pgrep","gitlab.*runner"]
          initialDelaySeconds: 10
          timeoutSeconds: 1
          periodSeconds: 10
          successThreshold: 1
          failureThreshold: 3
        ports:
        - name: "metrics"
          containerPort: 9252
        volumeMounts:
        - name: projected-secrets
          mountPath: /secrets
        - name: etc-gitlab-runner
          mountPath: /home/gitlab-runner/.gitlab-runner
        - name: configmaps
          mountPath: /configmaps
        resources:
          {}
      volumes:
      - name: runner-secrets
        emptyDir:
          medium: "Memory"
      - name: etc-gitlab-runner
        emptyDir:
          medium: "Memory"
      - name: projected-secrets
        projected:
          sources:
            - secret:
                name: "gitlab-runner"
                items:
                  - key: runner-registration-token
                    path: runner-registration-token
                  - key: runner-token
                    path: runner-token
      - name: configmaps
        configMap:
          name: gitlab-runner

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306

手动安装：
helm安装：https://blog.csdn.net/weixin_43606975/article/details/119944602?spm=1001.2014.3001.5501

#添加源
helm repo add gitlab https://charts.gitlab.io
1
2

egrep -v "#" gitlab-runn.yaml | grep -v "^$"
imagePullPolicy: IfNotPresent
#gitlab服务器地址
gitlabUrl: http://192.168.10.100/
#runner注册token
runnerRegistrationToken: "mNaxzsqyxpwQQCxJkSTa"  #从gitlab-runner获取的如下图
#当停止管道时等待其他作业终止时间
terminationGracePeriodSeconds: 3600
#最大并发作业数量
concurrent: 10
#新作业检查时隔
checkInterval: 30
sessionServer:
  enabled: false
rbac:
  create: true
  resources: ["pods", "pods/exec", "secrets"]
  verbs: ["get", "list", "watch", "create", "patch", "delete"]
  rules: []
  clusterWideAccess: false
  podSecurityPolicy:
    enabled: false
    resourceNames:
    - gitlab-runner
metrics:
  enabled: true
  portName: metrics
  port: 9252
  serviceMonitor:
    enabled: false
service:
  enabled: false
  type: ClusterIP
runners:
  config: |
    [[runners]]
      [runners.kubernetes]
        namespace = "{{.Release.Namespace}}"
        image = "ubuntu:16.04"
  #执行器类型
  executor: kubernetes
  #是否锁定false
  locked: false
  #你的tags
  tags: "k8s-runner,k8s"
  #是否运行没有标签的项目
  runUntagged: true
  #开启docker in docker
  privileged: true
  cache: {}
  builds: {}
  services: {}
  helpers: {}
securityContext:
  runAsUser: 100
  fsGroup: 65533
resources: {}
affinity: {}
nodeSelector: {}
tolerations: []
hostAliases: []
podAnnotations: {}
podLabels: {}
secrets: []
configMaps: {}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

helm template --namespace gitlab  gitlab-runner -f gitlab-runn.yaml gitlab/gitlab-runner > runner-manifest.yaml
#执行
kubectl  -f apply runner-manifest.yaml
1
2
3

k8s 1.23.1 会报错fsGroup的问题，删了就好。

7. 查看gitlab是否注册上
   
8. 配置连接k8s的api
   REGISTRY_USERNAME：镜像仓库用户名 root
   REGISTRY_PASSWORD：镜像仓库密码 123456789
   kube_config：KubeConfig的编码字符串

echo $(cat ~/.kube/config | base64) | tr -d " "
1

9. 配置.gitlab-ci.yml
   

#.gitlab-ci.yml
stages:
  - package
  - buidl
  - deploy

package:
    stage: package
    image: golang:1.17.8-alpine3.15
    tags: 
      - k8s
    script:
     - "go build -o gotest test.go"
     - "ls ./;pwd"
    only:
     - "tags"
    artifacts:             //下阶段可以使用此文件
      paths:
        - ./gotest

buidl:
    stage: buidl
    tags: 
      - k8s
    script:
     - "ls ./;pwd"
    only:
     - "tags"
     
deploy:
    stage: deploy
    image: registry.cn-hangzhou.aliyuncs.com/haoshuwei24/kubectl:1.16.6
    tags: 
      - k8s
    script:
     - mkdir -p $HOME/.kube
     - echo "$KUBERNETES_SECRET" >> "$HOME/.kube/config"
     - echo "$KUBERNETES_SECRET"
     - kubectl version
     - kubectl  get pods -A
    only:
     - "tags"  //用的tags，也可以用master,tages需要设置，master不需要。
     

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

#cat test.go
package main

import (
    "fmt"
)

func main() {
   // %d 表示整型数字，%s 表示字符串
    var stockcode=123
    var enddate="2020-12-31"
    var url="Code=%d&endDate=%s"
    var target_url=fmt.Sprintf(url,stockcode,enddate)
    fmt.Println(target_url)
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

9. 打tags自动发布
   
   
   
   

runner跑完了job任务就会自动删除。
kaniko-executor打包镜像后期添加，主要是没有镜像仓库。
k8s部署的gitlab问题：clone地址问题，不能解析到。临时解决方案：
进入容器修改地址，重启gitlab。

#kubectl  exec -it gitlab-58bd6f9c-w5hpb  bash
#cat /var/opt/gitlab/gitlab-rails/etc/gitlab.yml
...
  gitlab:
    ## Web server settings (note: host is the FQDN, do not include http://)
    host: 192.168.10.100
    port: 80
    https: false

...
#gitlab-ctl restart 
1
2
3
4
5
6
7
8
9
10
11

收工