安装k8s

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

# 主机名和固定ip
hostnamectl set-hostname master1
reboot

# 防火墙
ufw disable

# 更新一下
apt update -y
apt upgrade -y

# 相关组件
apt install selinux-utils policycoreutils ntp ntpdate htop nethogs nload tree lrzsz iotop iptraf-ng zip unzip ca-certificates curl gnupg build-essential gperf ipset ipvsadm -y

# selinux
vim /etc/selinux/config
SELINUX=disabled

# 时间同步
crontab -e
0 */1 * * * /usr/sbin/ntpdate time1.aliyun.com

# swap
vim /etc/fstab
# 注释掉swap

# 内核
vim /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0

# 模块加载
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
# 查看是否加载。出现了就是加载了
lsmod | grep br_netfilter
br_netfilter           22256  0
bridge                151336  1 br_netfilter
lsmod | grep overlay
overlay               151552  0

# 确认为1
sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward

# ipvs相关
# apt install ipset ipvsadm -y
# 加入
vim /etc/modules-load.d/k8s.conf 
overlay
br_netfilter
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

# 共享盘，可以忽略
mkdir /mnt/win 
mount -t cifs -o username="Everyone" //192.168.2.100/vm_share /mnt/win
cp /mnt/win/* .

# cri-containerd-cni版本会有问题  https://github.com/containerd/containerd/releases
tar Cxzvf /usr/local containerd-1.6.26-linux-amd64.tar.gz

# 试试systemctl daemon-reload
# 如果不行就找到它。放到/etc/systemd/system/下面重新daemon-reload
find / -name containerd.service
vim /etc/systemd/system/containerd.service

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target
[Service]
#uncomment to enable the experimental sbservice (sandboxed) version of containerd/cri integration
#Environment="ENABLE_CRI_SANDBOXES=sandboxed"
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999
[Install]
WantedBy=multi-user.target

mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml
vim /etc/containerd/config.toml
# https://github.com/containerd/containerd/issues/4203#issuecomment-651532765
systemdGroup = true
sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"

systemctl enable containerd --now


# 开始安装runc
# 安装c编译器和gperf
# apt install build-essential gperf -y
# 下载https://github.com/opencontainers/runc/releases libseccomp-2.5.4.tar.gz和runc.amd64
tar xf libseccomp-2.5.4.tar.gz
cd libseccomp-2.5.4/
./configure
make && make install
# 验证
find / -name "libseccomp.so"

chmod +x runc.amd64
把这个runc替换掉containerd的sbin里的runc
install -m 755 runc.amd64 /usr/local/sbin/runc

# 下载https://github.com/containernetworking/plugins/releases
mkdir -p /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.4.0.tgz

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# 阿里源
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg -o k8s.gpg
gpg --dearmor -o /etc/apt/keyrings/k8s.gpg k8s.gpg

cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb [signed-by=/etc/apt/keyrings/k8s.gpg] https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

apt update -y
apt install -y kubelet kubeadm kubectl

systemctl enable kubelet --now

1
2
3
4
5
6
7

# 初始化
kubeadm init --image-repository='registry.cn-hangzhou.aliyuncs.com/google_containers' \
--kubernetes-version=v1.28.2 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--apiserver-advertise-address=192.168.2.180 \
--cri-socket unix:///var/run/containerd/containerd.sock

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

apiVersion: kubeadm.k8s.io/v1beta3 # 不同版本的api版本不一样
# 节点加入时，认证token授权相关的基本信息。一般在内网比较安全，无需改动
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
# 第一个master节点配置的入口
localAPIEndpoint:
  advertiseAddress: 192.168.31.221 # 第一个master的ip
  bindPort: 6443
# node节点注册到master集群的通信方式
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: master1 # 第一个master的名字
  taints: null
  # taints: 
  #   - effect: NoSchedule
  #     key: node-role.kubernets.io/master
---
# 基本的集群属性信息
apiServer:
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
# 默认没有这个，多个master的时候，这里填keepalived的vip地址
# controlPlaneEndpoint: vip:6443
controllerManager: {}
dns: {}
# kubernetes的数据管理方式
etcd:
  local:
    dataDir: /var/lib/etcd
# 镜像仓库的配置
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kind: ClusterConfiguration
# kubernetes版本的定制
kubernetesVersion: 1.27.0
# kubernetes的网络基本信息
networking:
  dnsDomain: cluster.local
  serviceSubnet: "10.96.0.0/12"
  podSubnet: "10.244.0.0/16"
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs

1
2
3
4
5

# calico网络插件网段
# 这里应该和kubeadm init --pod-network-cidr=192.168.0.0/16 一致!!!
10.244.0.0/16
# https://docs.tigera.io/calico/latest/getting-started/kubernetes/quickstart
kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# This section includes base Calico installation configuration.
# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.Installation
apiVersion: operator.tigera.io/v1
kind: Installation
metadata:
  name: default
spec:
  # Configures Calico networking.
  calicoNetwork:
    # Note: The ipPools section cannot be modified post-install.
    ipPools:
    - blockSize: 26
      cidr: 10.244.0.0/16
      encapsulation: VXLANCrossSubnet
      natOutgoing: Enabled
      nodeSelector: all()
---
# This section configures the Calico API server.
# For more information, see: https://projectcalico.docs.tigera.io/master/reference/installation/api#operator.tigera.io/v1.APIServer
apiVersion: operator.tigera.io/v1
kind: APIServer
metadata:
  name: default
spec: {}

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17

kubectl edit configmap kube-proxy -n kube-system
mode: ipvs

kubectl rollout restart daemonset kube-proxy -n kube-system
# 或者一个一个来 kubectl delete po -n kube-system kube-proxy-xxx

# 验证使用了ipvs
kubectl logs kube-proxy-xxx -n kube-system | grep "Using ipvs Proxier"
# 新建service应该会有日志出现
kubectl logs -f -n kube-system -l k8s-app=kube-proxy

# ipvsadm可以帮助我们查看规则
apt install ipvsadm -y
# 查看所有规则
ipvsadm -Ln
# 出来的记录指向多个ip，正好就是service的endpoint里面的ip
ipvsadm -Ln | egrep -A2 'service的cluster-ip'

1
2

# 每个节点都会有一个 pod/calico-node 处于 running 状态
kubectl get all -A -o wide |grep calico

1

kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.8.1/deploy/static/provider/cloud/deploy.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# 冻结
kubectl cordon xxxx-node

# 驱离
# 会先让apiserver删除pod，然后被replicateSet发现后，写入新信息到etcd。然后etcd
kubectl drain xxx-node --force=true --ignore-daemonsets=true --delete-emptydir-data=true

# 删除节点
kubectl delete nodes xxx-node

# 在被移除节点上操作
kubeadm reset

# 清理资源
rm -rf /etc/cni/net.d/*
# master节点要删除 ~/.kube

# 可以跳过，直接到重启部分
tree /etc/kubernetes # 如果有文件就清理
# 重启一下
systemctl restart kubelet docker
# 如果有网络组件，应该清理掉
ifconfig
# 手动关闭网卡，如果是calico也使用一样的方法清理掉
ifconfig flannel.1 down
# 清理ip route
ip route list
ip route del 192.168.1.0/24
# 刷新缓存
ip route flush cache

# 重启一下
reboot

# 重新加入
kubeadm join ...

1
2

# 这是node节点的加入命令
kubeadm token create --print-join-command

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# 关闭防火墙
systemctl disable firewalld

# 关闭swap
swapon --show
# 在 /etc/sysctl.conf 中添加 vm.swappiness=0
echo "vm.swappiness=0" >> /etc/sysctl.conf
# 如果有内容，那么去 /etc/fstab 注释掉下面的东西
#/swap.img	none	swap	sw	0	0
# 重启电脑验证
swapon --show

# 关闭selinux
apt install selinux-utils
# 查看输出
getenforce

# 安装依赖
apt install socat conntrack ebtables ipset -y

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25

# 安装kubekey工具  
export KKZONE=cn  
curl -sfL https://get-kk.kubesphere.io | VERSION=v1.2.1 sh -  
chmod +x kk  
  
# 创建配置  
./kk create config  
# 编辑配置文件  
vim config-sample.yaml  
  
# 开始安装  
./kk create cluster -f config-sample.yaml --with-kubesphere  
# 确定没问题就yes 
  
# 如果上面没有--with-kubesphere，手动安装的话  
kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/kubesphere-installer.yaml  
kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/cluster-configuration.yaml  
# 检查安装日志  
kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f  
# 验证所有pod都起来了  
kubectl get pod --all-namespaces  
# 检查控制台端口，默认30880  
kubectl get svc/ks-console -n kubesphere-system  
# 访问nodeport   
ip:30880 admin/P@88w0rd

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

# 从集群创建配置文件
./kk create config --from-cluster
# 或
./kk create config -f kubeconfig配置文件

# 添加节点
# 编辑配置
./kk add nodes -f sample.yaml

# 删除节点
# 界面停止调度
# 编辑配置
./kk delete node <nodeName> -f config-sample.yaml

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

apiVersion: kubekey.kubesphere.io/v1alpha1
kind: Cluster
metadata:
  name: sample
spec:
  hosts:
  - {name: master, address: 192.168.10.71, internalAddress: 192.168.10.71, user: root, password: 密码}
  - {name: node1, address: 192.168.10.72, internalAddress: 192.168.10.72, user: root, password: 密码}
  - {name: node2, address: 192.168.10.73, internalAddress: 192.168.10.73, user: root, password: 密码}
  roleGroups:
    etcd:
    - master
    master:
    - master
    worker:
    - node1
    - node2
  controlPlaneEndpoint:
    ##Internal loadbalancer for apiservers
    #internalLoadbalancer: haproxy

    domain: lb.kubesphere.local
    address: ""
    port: 6443
  kubernetes:
    version: v1.21.5
    clusterName: cluster.local
  network:
    plugin: calico
    kubePodsCIDR: 10.233.64.0/18
    kubeServiceCIDR: 10.233.0.0/18
  registry:
    registryMirrors: ["https://u7hlore1.mirror.aliyuncs.com"]
    insecureRegistries: []
    privateRegistry: ""
  addons: []