Kubernetes高可用集群
0、服务器信息
服务器为腾讯云服务器（按需计费，按流量计费，并且将IP转换为弹性IP，使用结束可关机仅收取硬盘等固定资源费用）
| 服务器名称 | IP | 描述 |
|---|---|---|
| Kuernetes-1 | 49.235.103.143 | master1 |
| Kuernetes-2 | 49.234.36.76 | master2 |
| Kuernetes-3 | 42.192.36.218 | master3 |
| Kuernetes-4 | 42.192.46.16 | node1 |
| Kuernetes-5 | 49.235.123.219 | node2 |
| Kuernetes-6 | 49.234.36.130 | node3 |
| 网段 | 描述 |
|---|---|
| 10.19.0.0/16 | 机器内网 |
| 10.96.0.0/16 | service |
| 198.168.0.0/16 | pod |
1、环境准备
1.1、host配置
cat>>/etc/hosts<< EOF
49.235.103.143 Kuernetes-1
49.234.36.76 Kuernetes-2
42.192.36.218 Kuernetes-3
42.192.46.16 Kuernetes-4
49.235.123.219 Kuernetes-5
49.234.36.130 Kuernetes-6
EOF
1.2、ssh免密链接
# 生成ssh密钥
[root@Kuernetes-1 ~]# ssh-keygen -t rsa
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa.
Your public key has been saved in /root/.ssh/id_rsa.pub.
The key fingerprint is:
SHA256:5T64EqJDkJMHYMXBs4OqQGWKjtPbF5yvTfh0V3Ynpso root@Kuernetes-1
The key's randomart image is:
+---[RSA 2048]----+
|o.+o. |
|o * |
|.+= o . |
|*+.o o |
|== .. .S . = o|
|=.o . =. o = o.|
|oo + ..++ + o |
|. + . o=.+ + |
| . ..o+ E |
+----[SHA256]-----+
# 拷贝id_rsa.pub到目标机器，下次即可直接访问(第一次需要输入密码)
[root@Kuernetes-1 ~]# for i in Kuernetes-1 Kuernetes-2 Kuernetes-3 Kuernetes-4 Kuernetes-5 Kuernetes-6;do ssh-copy-id -i .ssh/id_rsa.pub $i;done
······
# 测试免密连接
[root@Kuernetes-1 ~]# ssh root@Kuernetes-2
Last login: Tue May 9 22:54:58 2023 from 183.195.73.137
[root@Kuernetes-2 ~]# hostname
Kuernetes-2
[root@Kuernetes-2 ~]# exit
logout
Connection to kuernetes-2 closed.
1.3、升级内核
# 查看内核(3.10内核在大规模集群具有不稳定性，需将内核升级到4.19+)
[root@Kuernetes-1 ~]# uname -sr
Linux 3.10.0-1160.88.1.el7.x86_64
# 升级软件包
[root@Kuernetes-1 ~]# yum update -y --exclude=kernel*
······
# 导入ELRepo软件仓库的公共秘钥
[root@Kuernetes-1 ~]# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
[root@Kuernetes-1 ~]# rpm -Uvh https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
Retrieving https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm
Preparing... ################################# [100%]
Updating / installing...
1:elrepo-release-7.0-6.el7.elrepo ################################# [100%]
# 镜像加速
[root@Kuernetes-1 ~]# yum install -y yum-plugin-fastestmirror
······
# 查看当前可升级到的版本
[root@Kuernetes-1 ~]# yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
······
# 升级内核(需要一定时间)
[root@Kuernetes-1 ~]# yum --enablerepo=elrepo-kernel install -y kernel-lt
······
# 查看当前系统已安装内核
[root@Kuernetes-1 ~]# awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux 7 Rescue 95a46ad0ee7f4772b6251edf4514b358 (5.4.242-1.el7.elrepo.x86_64)
1 : CentOS Linux (5.4.242-1.el7.elrepo.x86_64) 7 (Core)
2 : CentOS Linux (3.10.0-1160.88.1.el7.x86_64) 7 (Core)
3 : CentOS Linux (0-rescue-ba63ad6a0c7246dd8b30c727aae0e195) 7 (Core)
# 重构内核
[root@Kuernetes-1 ~]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.4.242-1.el7.elrepo.x86_64
Found initrd image: /boot/initramfs-5.4.242-1.el7.elrepo.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-1160.88.1.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-1160.88.1.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-95a46ad0ee7f4772b6251edf4514b358
Found initrd image: /boot/initramfs-0-rescue-95a46ad0ee7f4772b6251edf4514b358.img
Found linux image: /boot/vmlinuz-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195
Found initrd image: /boot/initramfs-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195.img
done
# 再次查看当前系统已安装内核
[root@Kuernetes-6 ~]# awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux (5.4.242-1.el7.elrepo.x86_64) 7 (Core)
1 : CentOS Linux (3.10.0-1160.88.1.el7.x86_64) 7 (Core)
2 : CentOS Linux (0-rescue-92581b1e8bfc4373a10654f47c8911f3) 7 (Core)
3 : CentOS Linux (0-rescue-ba63ad6a0c7246dd8b30c727aae0e195) 7 (Core)
# 修改默认内核（GRUB_DEFAULT=0）
[root@Kuernetes-1 ~]# vi /etc/default/grub
[root@Kuernetes-1 ~]# cat /etc/default/grub
GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=0
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL="serial console"
GRUB_TERMINAL_OUTPUT="serial console"
GRUB_CMDLINE_LINUX="crashkernel=2G-8G:256M,8G-16G:512M,16G-:768M console=ttyS0,115200 console=tty0 panic=5 net.ifnames=0 biosdevname=0 intel_idle.max_cstate=1 intel_pstate=disable processor.max_cstate=1 amd_iommu=on iommu=pt"
GRUB_DISABLE_RECOVERY="true"
GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"
# 再次重构内核
[root@Kuernetes-1 ~]# grub2-mkconfig -o /boot/grub2/grub.cfg
Generating grub configuration file ...
Found linux image: /boot/vmlinuz-5.4.242-1.el7.elrepo.x86_64
Found initrd image: /boot/initramfs-5.4.242-1.el7.elrepo.x86_64.img
Found linux image: /boot/vmlinuz-3.10.0-1160.88.1.el7.x86_64
Found initrd image: /boot/initramfs-3.10.0-1160.88.1.el7.x86_64.img
Found linux image: /boot/vmlinuz-0-rescue-95a46ad0ee7f4772b6251edf4514b358
Found initrd image: /boot/initramfs-0-rescue-95a46ad0ee7f4772b6251edf4514b358.img
Found linux image: /boot/vmlinuz-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195
Found initrd image: /boot/initramfs-0-rescue-ba63ad6a0c7246dd8b30c727aae0e195.img
done
# 重启
[root@Kuernetes-1 ~]# reboot
# 检查重启后的内核
[root@Kuernetes-1 ~]# uname -sr
Linux 5.4.242-1.el7.elrepo.x86_64
1.4、安装cfssl
https://github.com/cloudflare/cfssl/releases
[root@Kuernetes-1 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
······
[root@Kuernetes-1 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64
······
[root@Kuernetes-1 ~]# wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl-certinfo_1.6.4_linux_amd64
······
[root@Kuernetes-1 ~]# ls
cfssl_1.6.4_linux_amd64 cfssl-certinfo_1.6.4_linux_amd64 cfssljson_1.6.4_linux_amd64
[root@Kuernetes-1 ~]# chmod +x cfssl*
[root@Kuernetes-1 ~]# mv cfssl_1.6.4_linux_amd64 cfssl
[root@Kuernetes-1 ~]# mv cfssl-certinfo_1.6.4_linux_amd64 cfssl-certinfo
[root@Kuernetes-1 ~]# mv cfssljson_1.6.4_linux_amd64 cfssljson
[root@Kuernetes-1 ~]# ls -l
total 28572
-rwxr-xr-x 1 root root 12054528 Apr 11 03:07 cfssl
-rwxr-xr-x 1 root root 9560064 Apr 11 03:08 cfssl-certinfo
-rwxr-xr-x 1 root root 7643136 Apr 11 03:07 cfssljson
[root@Kuernetes-1 ~]# mv cfssl* /usr/bin/
[root@Kuernetes-1 ~]# cfssl
No command is given.
Usage:
Available commands:
revoke
bundle
certinfo
sign
genkey
ocspserve
info
crl
serve
gencert
gencsr
scan
gencrl
ocspdump
ocspsign
print-defaults
version
ocsprefresh
selfsign
Top-level flags:
1.5、系统设置
# set SELinux permissive(disable)
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
# close swap
swapoff -a && sysctl -w vm.swappiness=0
sed -ri 's/.*swap.*/#&/' /etc/fstab
ulimit -SHn 65535
# 在文件末尾追加内容
cat>>/etc/security/limits.conf<< EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF
1.6、ipvs组件
# 安装ipvs组件
[root@Kuernetes-1 ~]# yum install ipvsadm ipset sysstat conntrack libseccomp -y
······
# 修改ipvs配置
[root@Kuernetes-1 ~]# cat <<EOF | sudo tee /etc/modules-load.d/ipvs.conf
> ip_vs
> ip_vs_lc
> ip_vs_wlc
> ip_vs_rr
> ip_vs_wrr
> ip_vs_lblc
> ip_vs_lblcr
> ip_vs_dh
> ip_vs_sh
> ip_vs_fo
> ip_vs_nq
> ip_vs_sed
> ip_vs_ftp
> ip_vs_sh
> nf_conntrack
> ip_tables
> ip_set
> xt_set
> ipt_set
> ipt_rpfilter
> ipt_REJECT
> ipip
> EOF
# 重启
[root@Kuernetes-1 ~]# systemctl enable --now systemd-modules-load.service
Job for systemd-modules-load.service failed because the control process exited with error code. See "systemctl status systemd-modules-load.service" and "journalctl -xe" for details.
# 查看ipvs模块是否加载
[root@Kuernetes-1 ~]# lsmod | grep -e ip_vs -e nf_conntrack
ip_vs_ftp 16384 0
nf_nat 45056 1 ip_vs_ftp
ip_vs_sed 16384 0
ip_vs_nq 16384 0
ip_vs_fo 16384 0
ip_vs_sh 16384 0
ip_vs_dh 16384 0
ip_vs_lblcr 16384 0
ip_vs_lblc 16384 0
ip_vs_wrr 16384 0
ip_vs_rr 16384 0
ip_vs_wlc 16384 0
ip_vs_lc 16384 0
ip_vs 155648 24 ip_vs_wlc,ip_vs_rr,ip_vs_dh,ip_vs_lblcr,ip_vs_sh,ip_vs_fo,ip_vs_nq,ip_vs_lblc,ip_vs_wrr,ip_vs_lc,ip_vs_sed,ip_vs_ftp
nf_conntrack 147456 2 nf_nat,ip_vs
nf_defrag_ipv6 24576 2 nf_conntrack,ip_vs
nf_defrag_ipv4 16384 1 nf_conntrack
libcrc32c 16384 3 nf_conntrack,nf_nat,ip_vs
1.7、docker
#!/bin/bash
# remove old docker
yum remove docker \
docker-client \
docker-client-latest \
docker-common \
docker-latest \
docker-latest-logrotate \
docker-logrotate \
docker-engine
# install dependents
yum install -y yum-utils
# set yum repo
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# install docker
yum -y install docker-ce-20.10.9-3.el7 docker-ce-cli-20.10.9-3.el7 containerd.io
# start
systemctl enable docker --now
# docker config
sudo mkdir -p /etc/docker
sudo tee /etc/docker/daemon.json <<-'EOF'
{
"registry-mirrors": ["https://12sotewv.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
sudo systemctl daemon-reload
sudo systemctl restart docker
2、架构
