1.环境准备

root@cto-gpu-pro-n01:~# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 22.04.5 LTS
Release:        22.04
Codename:       jammy

1.1 关闭无用服务

• 关闭交换分区

# sed -i "/swap/{s/^/#/g}" /etc/fstab
# swapoff -a

• 关闭防火墙

root@boe:~# ufw disable
root@boe:~# ufw status
Status: inactive

1.2 环境和网络

• hostname

# hostname xxxx
# vim /etc/hostmame

• hosts

# cat >> /etc/hosts << EOF
10.10.xxx.47 t-master
10.10.xxx.46 t-node-01
10.10.xxx.45 t-node-02
10.10.xxx.44 t-node-03
EOF

• 打开路由

# cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1 
net.bridge.bridge-nf-call-iptables = 1 
net.ipv4.ip_forward = 1 
EOF
# modprobe br_netfilter 
# sysctl -p /etc/sysctl.d/k8s.conf 

• 加载内核模块

#cat > /etc/modules-load.d/ipvs.conf <<EOF 
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
nf_conntrack
EOF

# systemctl restart systemd-modules-load.service

验证结果如下：

root@boe:~# lsmod | grep '^ip_vs'
ip_vs_ftp              16384  0
ip_vs_sed              16384  0
ip_vs_nq               16384  0
ip_vs_fo               16384  0
ip_vs_sh               16384  0
ip_vs_dh               16384  0
ip_vs_lblcr            16384  0
ip_vs_lblc             16384  0
ip_vs_wrr              16384  0
ip_vs_rr               16384  1338
ip_vs_wlc              16384  0
ip_vs_lc               16384  0
ip_vs                 176128  1366 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp

1.3 apt源

• apt源的配置文件

vim /etc/apt/sources.list

• 内容如下

# See http://help.ubuntu.com/community/UpgradeNotes for how to upgrade to
# newer versions of the distribution.
deb http://archive.ubuntu.com/ubuntu/ jammy main restricted
# deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted

## Major bug fix updates produced after the final release of the
## distribution.
deb http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted

## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team. Also, please note that software in universe WILL NOT receive any
## review or updates from the Ubuntu security team.
deb http://archive.ubuntu.com/ubuntu/ jammy universe
# deb-src http://archive.ubuntu.com/ubuntu/ jammy universe
deb http://archive.ubuntu.com/ubuntu/ jammy-updates universe
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates universe

## N.B. software from this repository is ENTIRELY UNSUPPORTED by the Ubuntu
## team, and may not be under a free licence. Please satisfy yourself as to
## your rights to use the software. Also, please note that software in
## multiverse WILL NOT receive any review or updates from the Ubuntu
## security team.
deb http://archive.ubuntu.com/ubuntu/ jammy multiverse
# deb-src http://archive.ubuntu.com/ubuntu/ jammy multiverse
deb http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates multiverse

## N.B. software from this repository may not have been tested as
## extensively as that contained in the main release, although it includes
## newer versions of some applications which may provide useful features.
## Also, please note that software in backports WILL NOT receive any review
## or updates from the Ubuntu security team.
deb http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted universe multiverse

deb http://archive.ubuntu.com/ubuntu/ jammy-security main restricted
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-security main restricted
deb http://archive.ubuntu.com/ubuntu/ jammy-security universe
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-security universe
deb http://archive.ubuntu.com/ubuntu/ jammy-security multiverse
# deb-src http://archive.ubuntu.com/ubuntu/ jammy-security multiverse

• 安装依赖工具（非必要）

apt install vim wget curl net-tools  bind9-utils socat ipvsadm ipset

• vim: 用于安装 Vim 编辑器。
• wget: 用于下载文件的命令行工具。
• curl: 用于与服务器通信的命令行工具，常用于测试和下载数据。
• net-tools: 包含一些基本的网络工具，如 ifconfig，netstat 等。
• conntrack-tools: 包含用于管理和查看连接跟踪表的工具（如 conntrack）。
• bind9-utils: 提供一些与 DNS 相关的工具，如 dig，nslookup 等。
• socat: 用于处理网络和文件的双向数据流。
• ipvsadm: 用于配置 IP 虚拟服务器（IPVS）。
• ipset: 用于管理 IP 集合（用于防火墙等）。

1.4 系统优化

• 内核优化

# cat >>/etc/sysctl.conf <<EOF
net.ipv4.ip_forward = 1
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.tcp_max_syn_backlog = 65536
net.core.netdev_max_backlog =  32768
net.core.somaxconn = 32768
net.core.wmem_default = 8388608
net.core.rmem_default = 8388608
net.core.rmem_max = 16777216
net.core.wmem_max = 16777216
net.ipv4.tcp_timestamps = 0
net.ipv4.tcp_synack_retries = 2
net.ipv4.tcp_syn_retries = 2
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_mem = 94500000 915000000 927000000
net.ipv4.tcp_max_orphans = 3276800
net.ipv4.ip_local_port_range = 1024  65535
EOF

# sysctl -p

• 句柄数

ulimit -n 655350

永修生效修改如下两个文件

# cat >>/etc/security/limits.conf <<EOF
* soft memlock unlimited
* hard memlock unlimited
* soft nofile 655350
* hard nofile 655350
* soft nproc 655350
* hard nproc 655350
EOF

vim /etc/systemd/system.conf
DefaultLimitNOFILE=655350

或者

echo ulimit -n 655350 >>/etc/profile

2. 装containerd

• 需要的工具

sudo apt install apt-transport-https ca-certificates software-properties-common

• 添加秘钥

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg

• 添加apt源

echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null

• 更新索引

apt update

• 安装containerd

apt install containerd.io

• 生产containerd配置文件

mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml

• 添加 SystemdCgroup = true

# sed -i '/containerd.runtimes.runc.options/a\ \ \ \ \ \ \ \ \ \ \ \ SystemdCgroup = true' /etc/containerd/config.toml 

• 启动

systemctl start containerd
systemctl enable containerd

• 测试

ctr images pull docker.io/library/nginx:alpine

3. 接入k8s集群

3.1 kubelet、kubeadm、kubectl安装

• 使得 apt 支持 ssl 传输

apt-get update && apt-get install -y apt-transport-https

• 加载秘钥

curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -

• 添加k8s镜像源

cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

• apt安装

 apt-get install -y kubelet=1.23.17-00 kubeadm=1.23.17-00 kubectl=1.23.17-00

• runtime

crictl config runtime-endpoint /run/containerd/containerd.sock

• 启动服务

systemctl daemon-reload
systemctl enable kubelet && systemctl start kubelet 

3.2 缺少一个镜像

说明：集群中已经将需要的镜像改成我们内网镜像了，但是初始化的时候还是会需要检查registry.k8s.io/pause:3.8这个镜像，虽然pod启动时实际用到的仍是我们内网的pause:3.8镜像

• 拉取镜像

ctr images pull harbocto.boe.com.cn/kubernetes/pause:3.8

• 打tag

ctr images tag harbocto.boe.com.cn/kubernetes/pause:3.8 registry.k8s.io/pause:3.8

• 查看结果

[root@cto-gpu-pro-m01 ~]# crictl images
IMAGE                                                    TAG                 IMAGE ID            SIZE
......
harbocto.boe.com.cn/kubernetes/pause                     3.8                 4873874c08efc       309kB
registry.k8s.io/pause                                    3.8                 4873874c08efc       309kB
......

3.3 接入k8s集群

• master上

kubeadm token create --print-join-command

生成命令类似如下：

kubeadm join 10.10.xxx.xxx:7443 --token jzngpz.hw7emu8jv417xxk4 --discovery-token-ca-cert-hash sha256:1fcf23f0dc105037458567ea5xxxxxxx2950c0061bf2ec54adb7566810

• 在node节点上执行刚才生成的命令

4. 一些相关问题

• cni 网络创建是会生成这些二进制文件

root@cto-gpu-pro-n01:/opt/cni/bin# pwd
/opt/cni/bin
root@cto-gpu-pro-n01:/opt/cni/bin# ll
total 52408
drwxr-xr-x 2 root root    4096 Nov 14 20:24 ./
drwxr-xr-x 3 root root    4096 Nov 14 13:54 ../
-rwxr-xr-x 1 root root 2782728 Nov 14 20:24 bandwidth*
-rwxr-xr-x 1 root root 3104192 Nov 14 20:24 bridge*
-rwxr-xr-x 1 root root 7607056 Nov 14 20:24 dhcp*
-rwxr-xr-x 1 root root 2863024 Nov 14 20:24 dummy*
-rwxr-xr-x 1 root root 3165352 Nov 14 20:24 firewall*
-rwxr-xr-x 1 root root 2631704 Nov 14 20:24 flannel*
-rwxr-xr-x 1 root root 2775224 Nov 14 20:24 host-device*
-rwxr-xr-x 1 root root 2332792 Nov 14 20:24 host-local*
-rwxr-xr-x 1 root root 2871792 Nov 14 20:24 ipvlan*
-rwxr-xr-x 1 root root 2396976 Nov 14 20:24 loopback*
-rwxr-xr-x 1 root root 2893624 Nov 14 20:24 macvlan*
-rwxr-xr-x 1 root root 2689440 Nov 14 20:24 portmap*
-rwxr-xr-x 1 root root 3000032 Nov 14 20:24 ptp*
-rwxr-xr-x 1 root root 2542400 Nov 14 20:24 sbr*
-rwxr-xr-x 1 root root 2074072 Nov 14 20:24 static*
-rwxr-xr-x 1 root root 2456920 Nov 14 20:24 tuning*
-rwxr-xr-x 1 root root 2867512 Nov 14 20:24 vlan*
-rwxr-xr-x 1 root root 2566424 Nov 14 20:24 vrf*

• 一定不要忘记添加 SystemdCgroup = true

前边有方法，千万不要忘了。如果忘了，报错很模糊，不容易找到原因。

如果是docker的话需要在/etc/docker/daemon.json中添加：

{
    "exec-opts": ["native.cgroupdriver=cgroupfs"]
}

---