operator-sdk实战开发K8S CRD自定义资源对象_code-generate operator-sdk

环境说明

系统：CentOS Linux release 7.6.1810 (Core)
golang：v1.15
operator-sdk：v1.7.0
docker：v1.19 # 因为 operator-sdk 使用了多阶段构建功能，所以 docker 必须大于等于 v1.17 版本
k8s：k3s v1.20.6+k3s1 单节点

需求

定义一个 crd ，spec 包含以下信息：

Replicas	# 副本数
Image		# 镜像
Resources	# 资源限制
Envs		# 环境变量
Ports		# 服务端口
1
2
3
4
5

根据以上信息，controller 自动创建或者更新一个 deployment + service

注意：本实例只是展示了大概开发流程，故 crd 资源的 status 状态未做任何更新，实际编写中需要根据实际情况自行更新 status 状态

创建 APP

$ mkdir -p $GOPATH/src/github.com/zxl/app
$ cd $GOPATH/src/github.com/zxl/app
$ operator-sdk init --domain=example.com --repo=github.com/zxl/app
1
2
3

创建 API

$ operator-sdk create api --group app --version v1 --kind App --resource=true --controller=true
1

修改 CRD 类型定义代码 api/v1/app_types.go

/*
Copyright 2021.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1

import (
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

/*
修改定义后需要使用 make generate 生成新的 zz_generated.deepcopy.go 文件
 */

// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.

// AppSpec defines the desired state of App
type AppSpec struct {
	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
	// Important: Run "make" to regenerate code after modifying this file
	Replicas *int32               `json:"replicas"`		// 副本数
	Image    string               `json:"image"`		// 镜像
	Resources corev1.ResourceRequirements  `json:"resources,omitempty"`	// 资源限制
	Envs     []corev1.EnvVar      `json:"envs,omitempty"`	// 环境变量
	Ports    []corev1.ServicePort `json:"ports,omitempty"`	// 服务端口
}

// AppStatus defines the observed state of App
type AppStatus struct {
	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
	// Important: Run "make" to regenerate code after modifying this file
	//Conditions []AppCondition
	//Phase string
	appsv1.DeploymentStatus `json:",inline"`	// 直接引用 DeploymentStatus
}

//type AppCondition struct {
//	Type string
//	Message string
//	Reason string
//	Ready bool
//	LastUpdateTime metav1.Time
//	LastTransitionTime metav1.Time
//}

//+kubebuilder:object:root=true
//+kubebuilder:subresource:status

// App is the Schema for the apps API
type App struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   AppSpec   `json:"spec,omitempty"`
	Status AppStatus `json:"status,omitempty"`
}

//+kubebuilder:object:root=true

// AppList contains a list of App
type AppList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`
	Items           []App `json:"items"`
}

func init() {
	SchemeBuilder.Register(&App{}, &AppList{})
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

新增 resource/deployment/deployment.go

package deployment

import (
	appv1 "github.com/leffss/app/api/v1"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

func New(app *appv1.App) *appsv1.Deployment {
	labels := map[string]string{"app.example.com/v1": app.Name}
	selector := &metav1.LabelSelector{MatchLabels: labels}
	return &appsv1.Deployment{
		TypeMeta:   metav1.TypeMeta{
			APIVersion: "apps/v1",
			Kind: "Deployment",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: app.Name,
			Namespace: app.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(app, schema.GroupVersionKind{
					Group: appv1.GroupVersion.Group,
					Version: appv1.GroupVersion.Version,
					Kind: "App",
				}),
			},
		},
		Spec:       appsv1.DeploymentSpec{
			Replicas: app.Spec.Replicas,
			Selector: selector,
			Template: corev1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels: labels,
				},
				Spec: corev1.PodSpec{
					Containers: newContainers(app),
				},
			},
		},
	}
}

func newContainers(app *appv1.App) []corev1.Container  {
	var containerPorts []corev1.ContainerPort
	for _, servicePort := range app.Spec.Ports {
		var cport corev1.ContainerPort
		cport.ContainerPort = servicePort.TargetPort.IntVal
		containerPorts = append(containerPorts, cport)
	}
	return []corev1.Container{
		{
			Name:            app.Name,
			Image:           app.Spec.Image,
			Ports:           containerPorts,
			Env:             app.Spec.Envs,
			Resources:       app.Spec.Resources,
			ImagePullPolicy: corev1.PullIfNotPresent,
		},
	}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62

新增 resource/service/service.go

package service

import (
	appv1 "github.com/zxl/app/api/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

func New(app *appv1.App) *corev1.Service {
	return &corev1.Service{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Service",
			APIVersion: "v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:                       app.Name,
			Namespace: app.Namespace,
			OwnerReferences: []metav1.OwnerReference{
				*metav1.NewControllerRef(app, schema.GroupVersionKind{
					Group: appv1.GroupVersion.Group,
					Version: appv1.GroupVersion.Version,
					Kind: "App",
				}),
			},
		},
		Spec: corev1.ServiceSpec{
			Ports:                    app.Spec.Ports,
			Selector: map[string]string{
				"app.example.com/v1": app.Name,
			},
		},
	}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

修改 controller 代码 controllers/app_controller.go

/*
Copyright 2021.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
	"context"
	"encoding/json"
	"reflect"

	"github.com/zxl/app/resource/deployment"
	"github.com/zxl/app/resource/service"
	"k8s.io/apimachinery/pkg/api/errors"

	"github.com/go-logr/logr"
	"k8s.io/apimachinery/pkg/runtime"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"

	appv1 "github.com/zxl/app/api/v1"
	corev1 "k8s.io/api/core/v1"
	appsv1 "k8s.io/api/apps/v1"
)

// AppReconciler reconciles a App object
type AppReconciler struct {
	client.Client
	Log    logr.Logger
	Scheme *runtime.Scheme
}

//+kubebuilder:rbac:groups=app.example.com,resources=apps,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=app.example.com,resources=apps/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=app.example.com,resources=apps/finalizers,verbs=update

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the App object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.7.2/pkg/reconcile
func (r *AppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	_ = r.Log.WithValues("app", req.NamespacedName)
	// your logic here

	// 获取 crd 资源
	instance := &appv1.App{}
	if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil {
		if errors.IsNotFound(err) {
			return ctrl.Result{}, nil
		}
		return ctrl.Result{}, err
	}

	// crd 资源已经标记为删除
	if instance.DeletionTimestamp != nil {
		return ctrl.Result{}, nil
	}

	oldDeploy := &appsv1.Deployment{}
	if err := r.Client.Get(ctx, req.NamespacedName, oldDeploy); err != nil {
		// deployment 不存在，创建
		if errors.IsNotFound(err) {
			// 创建deployment
			if err := r.Client.Create(ctx, deployment.New(instance)); err != nil {
				return ctrl.Result{}, err
			}

			// 创建service
			if err := r.Client.Create(ctx, service.New(instance)); err != nil {
				return ctrl.Result{}, err
			}

			// 更新 crd 资源的 Annotations
			data, _ := json.Marshal(instance.Spec)
			if instance.Annotations != nil {
				instance.Annotations["spec"] = string(data)
			} else {
				instance.Annotations = map[string]string{"spec": string(data)}
			}
			if err := r.Client.Update(ctx, instance); err != nil {
				return ctrl.Result{}, err
			}
		} else {
			return  ctrl.Result{}, err
		}
	} else {
		// deployment 存在，更新
		oldSpec := appv1.AppSpec{}
		if err := json.Unmarshal([]byte(instance.Annotations["spec"]), &oldSpec); err != nil {
			return ctrl.Result{}, err
		}

		if !reflect.DeepEqual(instance.Spec, oldSpec) {
			// 更新deployment
			newDeploy := deployment.New(instance)
			oldDeploy.Spec = newDeploy.Spec
			if err := r.Client.Update(ctx, oldDeploy); err != nil {
				return ctrl.Result{}, err
			}

			// 更新service
			newService := service.New(instance)
			oldService := &corev1.Service{}
			if err := r.Client.Get(ctx, req.NamespacedName, oldService); err != nil {
				return ctrl.Result{}, err
			}
			clusterIP := oldService.Spec.ClusterIP	// 更新 service 必须设置老的 clusterIP
			oldService.Spec = newService.Spec
			oldService.Spec.ClusterIP = clusterIP
			if err := r.Client.Update(ctx, oldService); err != nil {
				return ctrl.Result{}, err
			}

			// 更新 crd 资源的 Annotations
			data, _ := json.Marshal(instance.Spec)
			if instance.Annotations != nil {
				instance.Annotations["spec"] = string(data)
			} else {
				instance.Annotations = map[string]string{"spec": string(data)}
			}
			if err := r.Client.Update(ctx, instance); err != nil {
				return ctrl.Result{}, err
			}
		}
	}
	return ctrl.Result{}, nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *AppReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&appv1.App{}).
		Complete(r)
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

修改 CRD 资源定义 config/samples/app_v1_app.yaml

apiVersion: app.example.com/v1
kind: App
metadata:
  name: app-sample
  namespace: default
spec:
  # Add fields here
  replicas: 2
  image: nginx:1.16.1
  ports:
  - targetPort: 80
    port: 8080
  envs:
  - name: DEMO
    value: app
  - name: GOPATH
    value: gopath
  resources:
    limits:
      cpu: 500m
      memory: 500Mi
    requests:
      cpu: 100m
      memory: 100Mi
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

修改 Dockerfile

# Build the manager binary
FROM golang:1.15 as builder

WORKDIR /workspace
# Copy the Go Modules manifests
COPY go.mod go.mod
COPY go.sum go.sum
# cache deps before building and copying source so that we don't need to re-download as much
# and so that source changes don't invalidate our downloaded layer

ENV GOPROXY https://goproxy.cn,direct

RUN go mod download

# Copy the go source
COPY main.go main.go
COPY api/ api/
COPY controllers/ controllers/
COPY resource/ resource/

# Build
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager main.go

# Use distroless as minimal base image to package the manager binary
# Refer to https://github.com/GoogleContainerTools/distroless for more details
#FROM gcr.io/distroless/static:nonroot
FROM kubeimages/distroless-static:latest
WORKDIR /
COPY --from=builder /workspace/manager .
USER 65532:65532

ENTRYPOINT ["/manager"]
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

• 添加了 goproxy 环境变量
• 新增 COPY 自定义的文件夹 resource
• gcr.io/distroless/static:nonroot 变更为 kubeimages/distroless-static:latest

部署运行

第一种：本地运行 controller

用于开发测试

app 项目根目录运行：

$ make generate && make manifests && make install && make run
1

• 本机需确保安装了 kubectl 工具，并且证书文件 ~/.kube/config 存在（保证为集群管理员权限）
• 测试完毕后使用 ctrl + c 停止程序，然后 make uninstall 删除 crd 定义

第二种：集群部署

1、make

$ make generate && make manifests && make install
1

2、构建镜像

$ make docker-build IMG=leffss/app:v1

$ docker images |grep app
leffss/app                v1                  1eaa4b6a4781        About a minute ago   46.5MB
1
2
3
4

3、准备镜像：

# 因为 k3s 安装的 k8s v1.20 默认使用的是 containerd，所以要导入镜像
$ docker save zxl/app:v1 > app.tar
$ ctr image import app.tar

$ docker pull kubesphere/kube-rbac-proxy:v0.8.0
$ docker tag kubesphere/kube-rbac-proxy:v0.8.0 gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
$ docker save gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0 > kube-rabc-proxy.tar
$ ctr image import kube-rabc-proxy.tar 
1
2
3
4
5
6
7
8

4、运行

$ make deploy IMG=leffss/app:v1
1

5、结果确认：

$ kubectl get service -A |grep app

$ kubectl -n app-system get pod

$ kubectl -n app-system get deployment
1
2
3
4
5

6、CRD 确认

$ kubectl get crd
NAME                              CREATED AT
addons.k3s.cattle.io              2021-05-02T01:03:34Z
helmcharts.helm.cattle.io         2021-05-02T01:03:34Z
helmchartconfigs.helm.cattle.io   2021-05-02T01:03:34Z
apps.app.example.com              2021-05-04T12:10:43Z
1
2
3
4
5
6

创建自定义资源

$ kubectl apply -f config/samples/app_v1_app.yaml 
app.app.example.com/app-sample created
1
2

查看控制台日志：

$ kubectl -n memcached-operator-system logs memcached-operator-controller-manager-6cf86db855-sqhpj -c manager

2021-05-05T19:56:32.798+0800    INFO    controller-runtime.metrics      metrics server is starting to listen      {"addr": ":8080"}
2021-05-05T19:56:32.798+0800    INFO    setup   starting manager
2021-05-05T19:56:32.799+0800    INFO    controller-runtime.manager      starting metrics server {"path": "/metrics"}
2021-05-05T19:56:32.800+0800    INFO    controller-runtime.manager.controller.app       Starting EventSource      {"reconciler group": "app.example.com", "reconciler kind": "App", "source": "kind source: /, Kind="}
2021-05-05T19:56:32.901+0800    INFO    controller-runtime.manager.controller.app       Starting Controller       {"reconciler group": "app.example.com", "reconciler kind": "App"}
2021-05-05T19:56:32.901+0800    INFO    controller-runtime.manager.controller.app       Starting workers {"reconciler group": "app.example.com", "reconciler kind": "App", "worker count": 1}
1
2
3
4
5
6
7
8

• 如果是第一种部署方式则直接查看控制台，第二种则查看集群中 app-controller-manager 的 pod 中的 manager 容器控制台日志

如果是第二种直接将 controller 部署到 k8s 集群的方式，可能会出现 RBAC 权限错误，解决方法是修改部署时的权限配置，这里我们使用最简单的方法是直接给 controller 绑定到 cluster-admin 集群管理员即可

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: cluster-admin-rolebinding
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: app-controller-manager
  namespace: app-system
1
2
3
4
5
6
7
8
9
10
11
12

删除 CRD 资源

$ kubectl apply -f config/samples/app_v1_app.yaml 
app.app.example.com/app-sample deleted
1
2

删除 CRD 定义

$ make uninstall
1

删除 controller

$ make undeploy
1

• 只适用第二种部署方式

总结

实际开发中只需要使用 operator-sdk（或者 kubebuilder）创建 CRD 以及相应的 controller，然后根据需求自行定义 crd 属性，并编写对应的 controller 逻辑代码，最终就可以实现一个完整的 operator。

补充：make deploy 含义

部署时使用 make deploy 实际是执行的什么命令呢？通过查看项目根目录 Makefile 可知实际运行命令为：

deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
     cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
     $(KUSTOMIZE) build config/default | kubectl apply -f -
1
2
3

• 实际上就是使用 kustomize 工具生成部署的 yaml 文件，然后使用 kubectl 工具应用

其他的 make install，make run 等命令也都在 Makefile 中有相关定义。

所以当在实际部署过程中，如果向手动部署的话，可以直接参考 Makefile 首先生成部署 yaml 文件，然后手动应用到 k8s 集群即可。