一、前期系统环境准备
准备3台主机：硬盘50G cpu2个 内存2G
1、3台主机同时配置
1）关闭防火墙与selinux、NetworkManager
[root@k8s-master ~]# systemctl stop firewalld [root@k8s-master ~]# systemctl disable firewalld Removed symlink /etc/systemd/system/multi-user.target.wants/firewalld.service. Removed symlink /etc/systemd/system/dbus-org.fedoraproject.FirewallD1.service. [root@k8s-master ~]# setenforce 0 [root@k8s-master ~]# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux [root@k8s-master ~]# sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config [root@k8s-master ~]# systemctl disable --now NetworkManager
2）配置yum源
[root@k8s-master yum.repos.d]# ls CentOS-Base.repo epel.repo docker-ce.repo epel-testing.repo kubernetes.repo [root@k8s-master ~]# yum clean all && yum makecache
3）配置主机映射
[root@k8s-master ~]# yum -y install vim [root@k8s-master ~]# vim /etc/hosts 10.0.0.66 k8s-master 10.0.0.77 k8s-node01 10.0.0.88 k8s-node02
4）配置主机间免密登录
[root@k8s-master ~]# ssh-keygen [root@k8s-master ~]# ssh-copy-id 10.0.0.77 [root@k8s-master ~]# ssh-copy-id 10.0.0.88
5）安装必备工具
[root@k8s-master ~]# yum install wget jq psmisc net-tools telnet yum-utils device-mapper-persistent-data lvm2 git tree -y
6）关闭swap 分区
[root@k8s-master ~]# swapoff -a && sysctl -w vm.swappiness=0 [root@k8s-master ~]# sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
7）同步时间
[root@k8s-master ~]# yum -y install ntpdate [root@k8s-master ~]# ntpdate time2.aliyun.com 4 Sep 10:08:59 ntpdate[1897]: adjust time server 203.107.6.88 offset 0.007780 sec [root@k8s-master ~]# which ntpdate /usr/sbin/ntpdate [root@k8s-master ~]# crontab -e * 5 * * * /usr/sbin/ntpdate time2.aliyun.com [root@k8s-master ~]# crontab -l * 5 * * * /usr/sbin/ntpdate time2.aliyun.com
8）配置 limit
[root@k8s-master ~]# ulimit -SHn 65535 [root@k8s-master ~]# vim /etc/security/limits.conf # 末尾添加如下内容 * soft nofile 65536 * hard nofile 131072 * soft nproc 65535 * hard nproc 655350 * soft memlock unlimited * hard memlock unlimited
2、只有master主机配置
1）安装 k8s ⾼可⽤性 Git 仓库并重启
# 在/root/⽬录下克隆⼀个名为k8s-ha-install.git的 Git仓库 [root@k8s-master ~]# cd /root/ ; git clone https://gitee.com/dukuan/k8s-ha-install.git [root@k8s-master ~]# ls anaconda-ks.cfg k8s-ha-install # 后续配置功能性pod的yaml文件 [root@k8s-master k8s-ha-install]# tree -L 2 . ├── calico.yaml ├── krm.yaml ├── LICENSE ├── metrics-server-0.3.7 │ └── components.yaml ├── metrics-server-3.6.1 │ ├── aggregated-metrics-reader.yaml │ ├── auth-delegator.yaml │ ├── auth-reader.yaml │ ├── metrics-apiservice.yaml │ ├── metrics-server-deployment.yaml │ ├── metrics-server-service.yaml │ └── resource-reader.yaml └── README.md 2 directories, 12 files
二、配置内核模块
1、3台主机同时配置
使用该工具可以同时操作多个主机
1）配置ipvs模块
[root@k8s-master ~]# yum install ipvsadm ipset sysstat conntrack libseccomp -y [root@k8s-master ~]# modprobe -- ip_vs [root@k8s-master ~]# modprobe -- ip_vs_rr [root@k8s-master ~]# modprobe -- ip_vs_wrr [root@k8s-master ~]# modprobe -- ip_vs_sh [root@k8s-master ~]# modprobe -- nf_conntrack # 在系统启动时加载下列 IPVS 和相关功能所需的模块 [root@k8s-master ~]# vim /etc/modules-load.d/ipvs.config ip_vs ip_vs_lc ip_vs_wlc ip_vs_rr ip_vs_wrr ip_vs_lblc ip_vs_lblcr ip_vs_dh ip_vs_sh ip_vs_fo ip_vs_nq ip_vs_sed ip_vs_ftp ip_vs_sh nf_conntrack ip_tables ip_set xt_set ipt_set ipt_rpfilter ipt_REJECT ipip # 加载系统内核参数并应用它们 [root@k8s-master ~]# sysctl --system # 开机⾃启systemd默认提供的⽹络管理服务 [root@k8s-master ~]# systemctl enable systemd-modules-load.service [root@k8s-master ~]# systemctl start systemd-modules-load.service # 在已加载的内核模块列表中查找与 ip_vs（IP Virtual Server，IP 虚拟服务器）和 nf_conntrack（Netfilter Connection Tracking，网络过滤器连接跟踪）相关的模块信息 [root@k8s-master ~]# lsmod | grep -e ip_vs -e nf_conntrack ip_vs_sh 12688 0 ip_vs_wrr 12697 0 ip_vs 141432 4 ip_vs_sh,ip_vs_wrr nf_conntrack 133053 1 ip_vs libcrc32c 12644 3 xfs,ip_vs,nf_conntrack
2）配置k8s内核
[root@k8s-master ~]# vim /etc/sysctl.d/k8s.conf net.bridge.bridge-nf-call-iptables = 1 net.bridge.bridge-nf-call-ip6tables = 1 fs.may_detach_mounts = 1 net.ipv4.conf.all.route_localnet = 1 vm.overcommit_memory=1 vm.panic_on_oom=0 fs.inotify.max_user_watches=89100 fs.file-max=52706963 fs.nr_open=52706963 net.netfilter.nf_conntrack_max=2310720 net.ipv4.tcp_keepalive_time = 600 net.ipv4.tcp_keepalive_probes = 3 net.ipv4.tcp_keepalive_intvl =15 net.ipv4.tcp_max_tw_buckets = 36000 net.ipv4.tcp_tw_reuse = 1 net.ipv4.tcp_max_orphans = 327680 net.ipv4.tcp_orphan_retries = 3 net.ipv4.tcp_syncookies = 1 net.ipv4.tcp_max_syn_backlog = 16384 net.ipv4.ip_conntrack_max = 65536 net.ipv4.tcp_max_syn_backlog = 16384 net.ipv4.tcp_timestamps = 0 # 保存后，所有节点重启，保证重启后内核依然加载 [root@k8s-master ~]# reboot
三、基本组件安装
1、3台主机同时配置
1）安装 Containerd
# 卸载之前的containerd [root@k8s-master ~]# yum remove -y podman runc containerd # 安装Docker和containerd [root@k8s-master ~]# yum install containerd.io docker-ce dockerce-cli -y [root@k8s-master ~]# yum list installed | grep docker containerd.io.x86_64 1.6.33-3.1.el7 @docker-ce-stable docker-buildx-plugin.x86_64 0.14.1-1.el7 @docker-ce-stable docker-ce.x86_64 3:26.1.4-1.el7 @docker-ce-stable docker-ce-cli.x86_64 1:26.1.4-1.el7 @docker-ce-stable docker-ce-rootless-extras.x86_64 26.1.4-1.el7 @docker-ce-stable docker-compose-plugin.x86_64 2.27.1-1.el7 @docker-ce-stable
2）配置 Containerd 所需模块
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf overlay br_netfilter EOF [root@k8s-master ~]# modprobe -- overlay [root@k8s-master ~]# modprobe -- br_netfilter
3）配置 Containerd 所需内核
[root@k8s-master ~]# cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 net.bridge.bridge-nf-call-ip6tables = 1 EOF [root@k8s-master ~]# sysctl --system
4）Containerd 配置⽂件
[root@k8s-master ~]# mkdir -p /etc/containerd # 读取containerd的配置并保存到/etc/containerd/config.toml [root@k8s-master ~]# containerd config default | tee /etc/containerd/config.toml [root@k8s-master ~]# vim /etc/containerd/config.toml # 找到第63行修改为sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9" # 找到containerd.runtimes.runc.options模块，添加SystemdCgroup = true，如果已经存在则直接修改（在第127行） # 添加sandbox_image = "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9"（第128行） # 加载systemctl控制脚本 [root@k8s-master ~]# systemctl daemon-reload # 启动containerd并设置开机启动 [root@k8s-master ~]# systemctl enable --now containerd Created symlink from /etc/systemd/system/multi-user.target.wants/containerd.service to /usr/lib/systemd/system/containerd.service.
5）配置 crictl 客户端连接的运⾏位置
# 配置容器运⾏环境的crictl.yml⽂件 [root@k8s-master ~]# cat <<EOF | sudo tee /etc/crictl.yaml runtime-endpoint: unix:///run/containerd/containerd.sock image-endpoint: unix:///run/containerd/containerd.sock timeout: 10 debug: false EOF
6）安装 Kubernetes 组件
# 安装 Kubeadm、Kubelet 和 Kubectl [root@k8s-master ~]# yum install kubeadm-1.28* kubelet-1.28* kubectl-1.28* -y [root@k8s-master ~]# systemctl daemon-reload [root@k8s-master ~]# systemctl enable --now kubelet Created symlink from /etc/systemd/system/multi-user.target.wants/kubelet.service to /usr/lib/systemd/system/kubelet.service. [root@k8s-master ~]# yum list installed | grep kube cri-tools.x86_64 1.26.0-0 @kubernetes kubeadm.x86_64 1.28.2-0 @kubernetes kubectl.x86_64 1.28.2-0 @kubernetes kubelet.x86_64 1.28.2-0 @kubernetes kubernetes-cni.x86_64 1.2.0-0 @kubernetes
问题解决：kubelet启动失败
# 查看日志 [root@k8s-master ~]# vim /var/log/messages # 配置文件未生成，重新安装kubelet
# 问题解决： [root@k8s-master ~]# yum -y remove kubelet [root@k8s-master ~]# yum -y install kubelet-1.28* [root@k8s-master ~]# systemctl start kubelet [root@k8s-master ~]# systemctl status kubelet Active: active (running) since 三 2024-09-11 14:25:57 CST; 3s ago # 由于kubeadm依赖kubelet所以卸载前者时后者也卸载了，需要重新安装 [root@k8s-master ~]# yum -y install kubeadm-1.28* # 查看kubelet端口是否启动 [root@k8s-master ~]# netstat -lntup | grep kube tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 2392/kubelet tcp6 0 0 :::10250 :::* LISTEN 2392/kubelet tcp6 0 0 :::10255 :::* LISTEN 2392/kubelet
2、只有master主机配置（Kubernetes 集群初始化）
1）Kubeadm 配置⽂件
[root@k8s-master ~]# vim kubeadm-config.yaml
# 粘贴文件内容并修改文件
# 修改第12行、24行、29行的ip地址为自己本机的ip地址
piVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: 7t2weq.bjbawausm0jaxury
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 10.0.0.66
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
name: k8s-master
taints:
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
---
apiServer:
certSANs:
- 10.0.0.66
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 10.0.0.66:6443
controllerManager: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.28.2
networking:
dnsDomain: cluster.local
podSubnet: 172.16.0.0/16
serviceSubnet: 10.96.0.0/16
scheduler: {}
# 将旧的kubeadm配置⽂件转换为新的格式
[root@k8s-master ~]# kubeadm config migrate --old-config kubeadm-config.yaml --new-config new.yaml
2）下载组件镜像
# 通过新的配置⽂件new.yaml从指定的阿⾥云仓库拉取kubernetes组件镜像 [root@k8s-master ~]# kubeadm config images pull --config /root/new.yaml
3）集群初始化
[root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs # 根据提示信息完成配置 [root@k8s-master ~]# mkdir -p $HOME/.kube [root@k8s-master ~]# sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config [root@k8s-master ~]# sudo chown $(id -u):$(id -g) $HOME/.kube/config # 将node结点加入集群的信息保存到一个文件中，以便使用 [root@k8s-master ~]# vim k8s.txt kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury \ --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99
问题解决：集群初始化失败
# 端口18258正被kubelet使用，初始化会自动启动kubelet，所以手动关闭kubelet服务 [root@k8s-master ~]# systemctl stop kubelet # 修改ip_forward文件内容 [root@k8s-master ~]# echo 1 > /proc/sys/net/ipv4/ip_forward [root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 错误信息显示本机内存不够，cpu数量不够，我们现在将本机内存提到4个G，cpu数量提到4个 # 注意要关闭本主机然后进行修改主机配置的操作 [root@k8s-master ~]# echo 1 > /proc/sys/net/ipv4/ip_forward [root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
# 检查kubelet为运行状态 [root@master ~]# systemctl status kubelet Active: active (running) since 五 2024-09-06 17:33:30 CST; 5min ago # 可能是配置文件的地址没有改，所以找不到主机，所以超时 [root@k8s-master ~]# vim new.yaml # 修改第12行、24行、29行的ip地址为自己本机的ip地址 # 初始化重置 [root@k8s-master ~]# kubeadm reset -f ; ipvsadm --clear ; rm -rf ~/.kube [root@k8s-master ~]# kubeadm init --config /root/new.yaml --upload-certs
4）加载环境变量
[root@k8s-master ~]# vim /root/.bashrc export KUBECONFIG=/etc/kubernetes/admin.conf [root@k8s-master ~]# source /root/.bashrc
5）查看组件容器状态
| 状态名称 | 中文 | 说明 |
|---|---|---|
| pending | 挂起 | 当前pod没有工作 |
| running | 运行中 | 当前pod正常工作 |
| containercreating | 正在创建容器 | 正在创建容器 |
[root@k8s-master ~]# kubectl get po -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 52m kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 52m kube-system etcd-k8s-master 1/1 Running 0 52m kube-system kube-apiserver-k8s-master 1/1 Running 0 52m kube-system kube-controller-manager-k8s-master 1/1 Running 0 52m kube-system kube-proxy-9r6st 1/1 Running 0 52m kube-system kube-proxy-lx5wz 1/1 Running 0 22m kube-system kube-proxy-xmk6s 1/1 Running 0 25m kube-system kube-scheduler-k8s-master 1/1 Running 0 52m
6）查看集群信息
[root@k8s-master ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master NotReady control-plane 25s v1.28.2
7）Token 过期处理
Token 过期后⽣成新的 token：
kubeadm token create --print-join-command
Master 需要⽣成 --certificate-key：
kubeadm init phase upload-certs --upload-certs
3、node结点执行
1）加入集群
[root@k8s-node01 ~]# kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury \ > --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99
问题解决：加入集群失败
# 端口被占用，手动停止kubelet，加入集群的过程中会自动启动 [root@k8s-node01 ~]# systemctl stop kubelet Warning: kubelet.service changed on disk. Run 'systemctl daemon-reload' to reload units. # 修改ip_forward文件 [root@k8s-node01 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward [root@k8s-node01 ~]# kubeadm join 10.0.0.66:6443 --token 7t2weq.bjbawausm0jaxury --discovery-token-ca-cert-hash sha256:f3ac431e03dae7f972728eb71eef1828264d42ec20a163893c812a2a0289cf99
4、master主机执行（Calico 组件安装）
1）查看集群状态与容器状态
[root@k8s-master ~]# kubectl get node NAME STATUS ROLES AGE VERSION k8s-master NotReady control-plane 31m v1.28.2 k8s-node01 NotReady <none> 4m4s v1.28.2 k8s-node02 NotReady <none> 57s v1.28.2 [root@k8s-master ~]# kubectl get po -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 52m kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 52m kube-system etcd-k8s-master 1/1 Running 0 52m kube-system kube-apiserver-k8s-master 1/1 Running 0 52m kube-system kube-controller-manager-k8s-master 1/1 Running 0 52m kube-system kube-proxy-9r6st 1/1 Running 0 52m kube-system kube-proxy-lx5wz 1/1 Running 0 22m kube-system kube-proxy-xmk6s 1/1 Running 0 25m kube-system kube-scheduler-k8s-master 1/1 Running 0 52m [root@k8s-master ~]# kubectl get po -Aowide NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 53m <none> <none> <none> <none> kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 53m <none> <none> <none> <none> kube-system etcd-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none> kube-system kube-apiserver-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none> kube-system kube-controller-manager-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none> kube-system kube-proxy-9r6st 1/1 Running 0 53m 10.0.0.66 k8s-master <none> <none> kube-system kube-proxy-lx5wz 1/1 Running 0 23m 10.0.0.88 k8s-node02 <none> <none> kube-system kube-proxy-xmk6s 1/1 Running 0 26m 10.0.0.77 k8s-node01 <none> <none> kube-system kube-scheduler-k8s-master 1/1 Running 0 54m 10.0.0.66 k8s-master <none> <none>
2）部署calico的pod
# 找到配置文件calico
[root@k8s-master ~]# cd k8s-ha-install/
# 切换 git 分⽀
[root@k8s-master k8s-ha-install]# git checkout manual-installation-v1.28.x
分支 manual-installation-v1.28.x 设置为跟踪来自 origin 的远程分支 manual-installation-v1.28.x。
切换到一个新分支 'manual-installation-v1.28.x'
# 修改 Pod ⽹段
[root@k8s-master k8s-ha-install]# ls
bootstrap CoreDNS dashboard metrics-server README.md
calico csi-hostpath kubeadm-metrics-server pki snapshotter
[root@k8s-master k8s-ha-install]# cd calico/
[root@k8s-master calico]# ls
calico.yaml
[root@k8s-master calico]# vim /etc/kubernetes/manifests/kube-controller-manager.yaml
# 获取已定义的Pod⽹段
[root@k8s-master calico]# POD_SUBNET=`cat /etc/kubernetes/manifests/kube-controller-manager.yaml | grep cluster-cidr= | awk -F= '{print $NF}'`
[root@k8s-master calico]# echo $POD_SUBNET
172.16.0.0/16
# 修改配置文件，将文件中的POD_CIDR替换成172.16.0.0/16
[root@k8s-master calico]# sed -i "s#POD_CIDR#${POD_SUBNET}#g" calico.yaml
# 创建pod
[root@k8s-master calico]# kubectl apply -f calico.yaml
3）查看容器状态
[root@k8s-master calico]# kubectl get po -A NAMESPACE NAME READY STATUS RESTARTS AGE kube-system calico-kube-controllers-6d48795585-v5d7x 0/1 Pending 0 69s kube-system calico-node-747k8 0/1 Init:0/3 0 69s kube-system calico-node-7klq9 0/1 Init:0/3 0 69s kube-system calico-node-j9b44 0/1 Init:0/3 0 69s kube-system coredns-6554b8b87f-2v4tx 0/1 Pending 0 104m kube-system coredns-6554b8b87f-zfqlb 0/1 Pending 0 104m kube-system etcd-k8s-master 1/1 Running 0 104m kube-system kube-apiserver-k8s-master 1/1 Running 0 104m kube-system kube-controller-manager-k8s-master 1/1 Running 1 (7m42s ago) 7m27s kube-system kube-proxy-9r6st 1/1 Running 0 104m kube-system kube-proxy-lx5wz 1/1 Running 0 74m kube-system kube-proxy-xmk6s 1/1 Running 0 77m kube-system kube-scheduler-k8s-master 1/1 Running 0 104m
