Centos kubeadm 部署 kubernetes 1.24.0 (单节点)（containerd & cri-docker 做 container runtime）_- --feature-gates=removeselflink=false

设置hostname

hostnamectl set-hostname k8s-master
#编辑hosts文件，添加hosts
vim /etc/hosts
172.21.16.7 k8s-master
1
2
3
4

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld
1
2

关闭selinux

setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
1
2

关闭swapoff分区

swapoff -a
sed -ri 's/.*swap.*/#&/' /etc/fstab
1
2

安装ipvsadm

yum install ipvsadm ipset sysstat conntrack libseccomp -y

cat >> /etc/modules-load.d/ipvs.conf <<EOF 
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF

systemctl restart systemd-modules-load.service
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

配置前置需求

#配置前置需求
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

systemctl restart systemd-modules-load.service

# sysctl params required by setup, params persist across reboots
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF
# Apply sysctl params without reboot
sudo sysctl --system

##保险执行一下命令
sysctl -w net.ipv4.ip_forward=1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22

安装 container runtime

containerd

sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo yum install containerd.io
#初始化默认配置
containerd config default | tee /etc/containerd/config.toml

#修改containerd配置更改cgroup
sed -i "s#SystemdCgroup\ \=\ false#SystemdCgroup\ \=\ true#g" /etc/containerd/config.toml
#修改镜像源
sed -i "s#k8s.gcr.io#registry.aliyuncs.com/google_containers#g"  /etc/containerd/config.toml


#安装cni ,docker 源安装 containerd 需要手动安装cni
wget https://github.com/containernetworking/plugins/releases/download/v1.1.1/cni-plugins-linux-amd64-v1.1.1.tgz
mkdir -p /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.1.1.tgz

systemctl daemon-reload
systemctl enable --now containerd

#安装crictl
VERSION="v1.24.1"
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-$VERSION-linux-amd64.tar.gz
sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
rm -f crictl-$VERSION-linux-amd64.tar.gz

#配置crictl
cat <<EOF | tee /etc/crictl.yaml
runtime-endpoint: "unix:///run/containerd/containerd.sock"
image-endpoint: "unix:///run/containerd/containerd.sock"
timeout: 10
debug: false
pull-image-on-create: false
disable-pull-on-run: false
EOF
systemctl restart  containerd
crictl info
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

cri-docker

#安装配置docker
yum remove docker docker-client docker-client-latest docker-common  docker-latest docker-latest-logrotate docker-logrotate docker-engine
sudo yum install -y yum-utils device-mapper-persistent-data  lvm2
sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
sudo yum install docker-ce docker-ce-cli containerd.io
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
	"exec-opts": ["native.cgroupdriver=systemd"],
	"log-driver": "json-file",
	"log-opts": {
		"max-size": "100m"
	},
	"experimental": false,
	"debug": false,
	"max-concurrent-downloads": 10,
	"registry-mirrors": ["https://a7h8080e.mirror.aliyuncs.com"]
}
EOF
sudo systemctl daemon-reload
sudo systemctl enable docker
sudo systemctl restart docker

#安装 golang
yum install golang
#安装 git
yum install git

#安装 cri-docker
git clone https://github.com/Mirantis/cri-dockerd.git
cd cri-dockerd
mkdir bin
go build -o bin/cri-dockerd
mkdir -p /usr/local/bin
install -o root -g root -m 0755 bin/cri-dockerd /usr/local/bin/cri-dockerd

#更改 packaging/systemd 目录下 cri-docker.service 和 cri-docker.socket 文件
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// 
改为：
ExecStart=/usr/bin/cri-dockerd --container-runtime-endpoint fd:// --network-plugin=cni --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.8

ListenStream 的值
改为：
ListenStream=/var/run/cri-dockerd.sock

cp -a packaging/systemd/* /etc/systemd/system
sed -i -e 's,/usr/bin/cri-dockerd,/usr/local/bin/cri-dockerd,' /etc/systemd/system/cri-docker.service
systemctl daemon-reload
systemctl enable cri-docker.service
systemctl restart cri-docker.socket

crictl config runtime-endpoint unix:///run/cri-dockerd.sock
crictl config image-endpoint unix:///run/cri-dockerd.sock
#或者编辑 vim /etc/crictl.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

安装kubeadm ,kubectl ,kubelet

# 添加kubernetes yum软件源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

sudo yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
sudo systemctl enable --now kubelet
# 命令补全
yum install -y bash-completion
source <(crictl completion bash)
crictl completion bash >/etc/bash_completion.d/crictl
source <(kubectl completion bash)
kubectl completion bash >/etc/bash_completion.d/kubectl
source /usr/share/bash-completion/bash_completion

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

配置kubeadm

kubeadm config print init-defaults > kubeadm.yaml
1

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 172.21.16.7
  bindPort: 6443
nodeRegistration:
# 如果用cri-docker
 #criSocket: unix:///var/run/cri-dockerd.sock
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: k8s-master
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: 1.24.0
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16
scheduler: {}

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

创建集群

kubeadm init --config kubeadm.yaml

mkdir -p $HOME/.kube && \
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && \
chown $(id -u):$(id -g) $HOME/.kube/config
# 下载calico 网络配置
curl -O https://docs.projectcalico.org/manifests/calico.yaml

#初始化网络配置
kubectl apply -f calico.yaml
#设置端口范围 vim /etc/kubernetes/manifests/kube-apiserver.yaml 添加
- --service-node-port-range=1-65535
1
2
3
4
5
6
7
8
9
10
11
12

单节点去除master节点的污点

kubectl taint node k8s-master node-role.kubernetes.io/control-plane-
1

补充k8s 1.24.0版本基于nfs-client-provisioner 的StorageClass PVC一直 pending，无法创建`

旧版本的k8s 解决方案是更改 /etc/kubernetes/manifests/kube-apiserver.yaml ,添加

- --feature-gates=RemoveSelfLink=false
1

新版本后(1.24)更改此配置导致kubelet无法启动，使用新版本的nfs-client-provisioner 即可解决此问题

#添加repo
helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
#拉去资源
helm pull nfs-subdir-external-provisioner/nfs-subdir-external-provisioner
 
1
2
3
4
5

更改配置

# 由于国内网络拉取不到谷歌仓库，这里在docker hub上找的资源
image:
  repository: docker.io/willdockerhub/nfs-subdir-external-provisioner
...

nfs:
  server: xx.xx.xx.xxx
  path: xx
...
1
2
3
4
5
6
7
8
9

Tip1: kubernetes 1.24之后创建 serviceaccount 不会创建secrets了，所以需要service-account-token 来做 bearer token得情况应该使用TokenRequest 的方式来获取token

1. 创建一个serviceaccount
kubectl create sa admin
2. 给sa绑定一个clusterrole
kubectl create clusterrolebinding admin --clusterrole=cluster-admin --serviceaccount=test:admin
3. 从sa创建个token
kubectl create token admin
1
2
3
4
5
6