综合实验
实验目的:
静态资源和动态资源分别存放在远端存储NFS上,NFS上数据实现实时备份,用户通过负载访问后端的web服务。实现ngixn负载高可用,当keepalived master宕机,vip能自动跳转到备用节点
实验环境:
六台服务器都是centos8.5系统
主机名 | IP地址 |
---|---|
master-kpa1 | 10.1.1.161 |
backup-kpa2 | 10.1.1.162 |
nginx-web1 | 10.1.1.121 |
nginx-web2 | 10.1.1.122 |
master-nfs | 10.1.1.123 |
slave-nfs | 10.1.1.124 |
实验步骤:
1NFS
1.1nfs共享和实时同步
#10.1.1.123master-nfs服务器端
#10.1.1.121master-nfs服务器端
(1)#在nfs服务器端安装nfs-utils和rpcbind包
[root@master-nfs ~]# yum install -y nfs-utils rpcbind
#nfs-utils:提供了NFS服务器程序和对应的管理工具
#rpcbind:获取nfs服务器端的端口等信息
[root@master-nfs ~]# systemctl start rpcbind
[root@master-nfs ~]# yum -y install net-tools #此包中含有一些常用网络查看命令
[root@master-nfs ~]# netstat -tunlp | grep 111
(2)#创建/data/NFSdata目录,更改属主、属组
[root@master-nfs ~]# mkdir -p /data/NFSdata/web1
[root@master-nfs ~]# mkdir -p /data/NFSdata/web2
(3)#注意:此处不改权限,客户端没有创建文件权限
[root@master-nfs ~]# chown -R nobody:nobody /data
(4)#配置NFS服务的配置文件
[root@master-nfs ~]# vim /etc/exports
/data/NFSdata/web1 #表示要共享文件的目录
10.1.1.0/24 #表示所有允许访问的客户端IP网段
(rw,sync) #rw:表示读写权限,sync:表示数据同步写入内存硬盘
/data/NFSdata/web2 10.1.1.0/24(rw,sync)
/data/NFSdata/web1 10.1.1.0/24(rw,sync)
(5)#重启服务及实现开机自启动
[root@master-nfs ~]# systemctl start nfs-server.service
[root@master-nfs ~]# systemctl enable rpcbind.service
[root@master-nfs ~]# systemctl enable nfs-server.service --now
1.2配置nfs客户端
2.1nginx-web1上10.1.1.121配置nfs客户端
(1)下载工具包nfs-utils
[root@nginx-web1 ~]# yum -y install nfs-utils
(2)#查看远程主机的NFS共享
[root@nginx-web1 ~]# showmount -e 10.1.1.123
Export list for 10.1.1.123:
/data/NFSdata/web1 10.1.1.0/24
/data/NFSdata/web2 10.1.1.0/24
(3)创建挂载目录
[root@nginx-web1 ~]# mkdir -p /data-web1/static
[root@nginx-web1 ~]# mkdir -p /data-web1/image
(4)永久挂载
[root@nginx-web1 ~]# vim /etc/fsstab/
10.1.1.123:/data/NFSdata/web1 /data-web1 nfs defaults 0 0
[root@nginx-web1 ~]# mount -a#挂载生效
(5)测试是否共享
[root@nginx-web1 static]# touch 1.txt
[root@nginx-web1 static]# ls
1.txt index.html
[root@master-nfs web1]# cd /data/NFSdata/web1/static/
[root@master-nfs static]# ls
1.txt index.html
2.2nginx-web2上10.1.1.122配置nfs客户端
(1)下载工具包nfs-utils
[root@nginx-web2 ~]# yum -y install nfs-utils
(2)#查看远程主机的NFS共享
[root@nginx-web2 ~]# showmount -e 10.1.1.123
Export list for 10.1.1.123:
/data/NFSdata/web1 10.1.1.0/24
/data/NFSdata/web2 10.1.1.0/24
(3)创建挂载目录
[root@nginx-web2 ~]# mkdir -p /data-web2/static
[root@nginx-web2 ~]# mkdir -p /data-web2/image
(4)永久挂载
[root@nginx-web2 ~]# vim /etc/fsstab/
10.1.1.123:/data/NFSdata/web2 /data-web1 nfs defaults 0 0
[root@nginx-web2 ~]# mount -a#挂载生效
(5)测试是否共享
[root@nginx-web2 ~]# cd /data-web2/static
[root@nginx-web2 static]# ls
index.html
[root@nginx-web2 static]# touch 2.txt
[root@nginx-web2 static]# ls
2.txt index.html
[root@master-nfs ~]# cd /data/NFSdata/web2/static/
[root@master-nfs static]# ls
2.txt index.html
1.3部署Rsync服务
在10.1.1.124slave-nfs上部署Rsync服务端
(1)下载Rsync软件包
[root@slave-nfs ~]#yum -y install rsync
(2)新增vim /etc/rsyncd.conf配置文件
[root@slave-nfs ~]# vim /etc/rsyncd.conf
[root@slave-nfs ~]# cat /etc/rsyncd.conf
uid = rsync
#组id
gid = rsync
#程序安全设置
use chroot = no
#客户端连接数
max connections = 200
#进程号文件位置
pid file = /var/run/rsyncd.pid
#进程锁文件位置
lock file = /var/run/rsync.lock
#日志文件位置
log file = /var/run/rsyncd.log
#连接超时时间
timeout = 300
#3.1版本以上要加这个
fake super = yes
#模块名称
[backup]
#同步数据的目录
path = /backup
#有错误时忽略
ignore errors
#只读模式(true为只读,false为可读可写)
read only = false
#阻止远程列表
list = false
#允许访问的IP
hosts allow = 10.1.1.0/24
#虚拟用户
auth users = rsync_backup
#存放用户和密码的文件
secrets file = /etc/rsync.password
(3)创建密码文件vi /etc/rsync.password
[root@slave-nfs ~]# vim /etc/rsync.password
rsync_backup:123456
(4)给/etc/rsync.password降权
[root@slave-nfs ~]# chmod 600 /etc/rsync.password
(5)创建程序用户rsync
[root@slave-nfs ~]# useradd -M -s /sbin/nologin rsync
(6)创建/backup目录并修改所有者所属组
[root@slave-nfs ~]# mkdir /backup
[root@slave-nfs ~]# chown rsync.rsync /backup
(7)守护进程启动rsync
[root@slave-nfs ~]# rsync --daemon
master-nfs10.1.1.123作为rsync客户端
(1)123客户端节点新增密码文件vim /etc/rsync.password
[root@master-nfs ~]# vim /etc/rsync.password
123456
(2)给/etc/rsync.password降权
[root@master-nfs ~]# chmod 600 /etc/rsync.password
(3)测试123节点传输文件到124上
[root@master-nfs ~]# touch 121.txt
[root@master-nfs ~]# rsync -zav 121.txt rsync_backup@10.1.1.124::backup --password-file=/etc/rsync.password
sending incremental file list
121.txt
sent 90 bytes received 43 bytes 88.67 bytes/sec
total size is 0 speedup is 0.00
[root@slave-nfs backup]# ls
121.txt 2.txt
#-a:表示以归档模式同步文件,相当于 参数的缩写。这个选项会保留文件的元数据(如所有者、权限、时间戳等)以及其他有用的信息,例如符号链接和设备文件。-rlptgoD
-v:表示启用详细模式,输出同步过程中的详细信息。
-z:表示使用压缩算法进行传输,可以减少数据传输量。在网络较慢或传输大文件时特别有用。
rsync -avz 命令可以将文件或目录以归档模式进行同步,并在同步过程中输出详细信息,同时使用压缩算法减少传输量。
1.4部署inotify服务
NFS服务器上部署inotify服务,实现实时同步
(1)#下载epel源
[root@master-nfs ~]# yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm
(2)#清除缓存
[root@master-nfs ~]# yum clean all
[root@master-nfs ~]# yum makecache
(3)下载inotify包
[root@master-nfs ~]# yum -y install inotify-tools
(4)编写脚本inotify脚本
[root@master-nfs ~]# vim inotify.sh
backupServer=10.1.1.124
path=/data/NFSdata/
inotifywait -mrq --format '%w%f' -e create,close_write,delete $path | while read line
do
if [ -f $line ];then
rsync -za $line --delete rsync_backup@$backupServer::backup --password-file=/etc/rsync.password
else
cd $path
rsync -za ./ --delete rsync_backup@$backupServer::backup --password-file=/etc/rsync.password
fi
done
(5)#后台运行
[root@master-nfs ~]# sh inotify.sh &
[1] 350
2.实现nignx负载
2.1安装Nignx
四台服务器都安装nginx
yum -y install nginx
systemctl enable nginx.service --now
systemctl status nginx.service
cd /etc/nginx/
cp nginx.conf nginx.conf.bak
2.2配置负载
[root@master-kpa1 ~]# vim /etc/nginx/nginx.conf
upstream myweb {
server 10.1.1.121:80 weight=1;
server 10.1.1.122:80 weight=1;
}
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name kpa1.sxh.com;
location / {
root /usr/share/nginx/html;
proxy_pass http://myweb;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
[root@backup-kpa2 ~]# vim /etc/nginx/nginx.conf
upstream myweb {
server 10.1.1.121:80 weight=1;
server 10.1.1.122:80 weight=1;
}
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name kpa1.sxh.com;
location / {
root /usr/share/nginx/html;
proxy_pass http://myweb;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
2.3配置动静分离
[root@nginx-web1 ~]# vim /etc/nginx/nginx.conf
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name kpa1.sxh.com;
location / {
root /data-web1/static;
}
#location /sxh1.jpg {
#root /data-web1/image;
#}
location ~* \.(gif|jpg|jpeg)$ {
root /data-web1/image;
}
[root@nginx-web2 ~]# vim /etc/nginx/nginx.conf
server {
listen 80 default_server;
listen [::]:80 default_server;
server_name kpa1.sxh.com;
location / {
root /data/static;
}
#location /sxh2.jpg {
#root /data/image;
#}
location ~* \.(gif|jpg|jpeg)$ {
root /data-web1/image;
}
2.4测试反向代理
[root@slave-nfs ~]# curl kpa1.sxh.com
this is a papge web1 10.1.1.121
[root@slave-nfs ~]# curl kpa1.sxh.com
this a page web2 10.1.1.122
3.实现keepalived
3.1编译安装kpa
master-kpa1和backup-kpa2都需要编译安装keepalived
#下载依赖软件
[root@master-kpa1 ~]# yum install gcc curl openssl-devel libnl3-devel net-snmp-devel
#下载二进制文件
[root@master-kpa1 ~]# wget https://keepalived.org/software/keepalived-2.0.20.tar.gz
#解压到指定目录
[root@master-kpa1 ~]# tar xvf keepalived-2.0.20.tar.gz -C /usr/local/src
#选项--disable-fwmark 可用于禁用iptables规则,可访止VIP无法访问,无此选项默认会启用iptables规则
[root@master-kpa1 ~]# cd /usr/local/src/keepalived-2.0.20/
#配置文件路径
[root@master-kpa1 keepalived-2.0.20]# ./configure --prefix=/usr/local/keepalived --disable-fwmark
#编译并安装
[root@master-kpa1 keepalived-2.0.20]# make && make install
[root@master-kpa1 keepalived-2.0.20]# cd
[root@master-kpa1 ~]# /usr/local/keepalived/sbin/keepalived -v
#创建配置文件
[root@master-kpa1 ~]# mkdir /etc/keepalived #没有创建,则服务起不来
[root@master-kpa1 ~]# cp /usr/local/keepalived/etc/keepalived/keepalived.conf /etc/keepalived
[root@master-kpa1 ~]## systemctl enable --now keepalived.service
#注意事项
#不进行下面配置,结果:重启不报错,但是status状态一直dead
[root@master-kpa1 ~]# vim /etc/keepalived/keepalived.conf
vrrp_instance VI_1 {
state MASTER
interface ens160#根据自己网卡名设置
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.1.1.88 #改为vip
}
}
[root@master-kpa1 ~]# systemctl restart keepalived.service
[root@master-kpa1 ~]# systemctl stauts keepalived.service
3.2实现kpa单主架构
master-kpa1配置
[root@master-kpa1 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
3059955740@qq.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id master-kpa1
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 230.1.1.1
}
vrrp_script check_nginx {
script "/usr/bin/killall -0 nginx"
interval 3
weight -50
fail 3
rise 1
}
vrrp_instance VI_1 {
state MASTER
interface ens160
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.1.1.88 dev ens160 label ens160:1
}
notify_master "/usr/bin/systemctl restart nginx.service"
notify_backup "/usr/bin/systemctl restart nginx.service"
# notify_master "/etc/keepalived/notify.sh master"
# notify_backup "/etc/keepalived/notify.sh backup"
# notify_fault "/etc/keepalived/notify.sh fault"
track_script {
check_nginx
}
}
backup-kpa2配置
[root@backup-kpa2 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
2923035330@qq.com
}
notification_email_from keepalived@localhost
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id back-kpa2
vrrp_skip_check_adv_addr
vrrp_garp_interval 0
vrrp_gna_interval 0
vrrp_mcast_group4 230.1.1.1
}
vrrp_script check_nginx {
script "/usr/bin/killall -0 nginx"
interval 3
weight -50
fail 3
rise 1
}
vrrp_instance VI_1 {
state BACKUP
interface ens160
virtual_router_id 51
priority 70
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.1.1.88 dev ens160 label ens160:2
}
track_interface {
ens160
}
notify_master "/usr/bin/systemctl restart nginx.service"
notify_backup "/usr/bin/systemctl restart nginx.service"
#notify_master "/etc/keepalived/notify.sh master"
#notify_backup "/etc/keepalived/notify.sh backup"
#notify_fault "/etc/keepalived/notify.sh fault"
track_script {
check_nginx
}
}
抓包分析
[root@kpa1 ~]# tcpdump -i ens160 -nn src host 10.1.1.161 and dst 10.1.1.162
dropped privs to tcpdump
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on ens160, link-type EN10MB (Ethernet), capture size 262144 bytes
15:52:48.226434 IP 10.1.1.161 > 10.1.1.162: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
15:52:49.227488 IP 10.1.1.161 > 10.1.1.162: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
15:52:50.228497 IP 10.1.1.161 > 10.1.1.162: VRRPv2, Advertisement, vrid 66, prio 100, authtype simple, intvl 1s, length 20
3.3QQ邮箱设置
[root@kpa1 ~]# vim /etc/mail.rc
set from=3059955740@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=3059955740@qq.com
set smtp-auth-password=zoboduhoqcqcdfhf
set smtp-auth=login
[root@kpa1 ~]# yum -y install mailx
#发送邮件测试
[root@kpa1 ~]# echo "Test Mail 30599555740" |mail -s warning 3059955740@qq.com
3.4创建通知脚本
[root@kpa1 ~]# cat /etc/keepalived/notify.sh
#!/bin/bash
contact='3059955740@qq.com'
notify() {
mailsubject="(hostname) to be $1,vip floating"
mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
echo "$mailbody" |mail -s "$mailsubject" $contact
}
case $1 in
master)
notify master
;;
backup)
notify backup
;;
fault)
notify fault
;;
*)
echo "Usage: $(basename $0) {master|backup|fault}"
exit 1
;;
esac
[root@kpa1 ~]# chmod a+x /etc/keepalived/notify.sh
3.5kpa测试宕机
[root@master-kpa1 ~]# killall keepalived
[root@master-kpa1 ~]# systemctl restart keepalived.service