1.环境说明:
系统 | 主机名 | IP地址 | 内存 | 添加共享磁盘大小 |
---|---|---|---|---|
Centos7.9 | gpfs1 | 192.168.10.101 | 2G | 20G |
Centos7.9 | gpfs2 | 192.168.10.102 | 2G | 20G |
2.配置共享硬盘:
-
前提:两台虚拟机没有拍摄快照
-
在mds001主机中:
-
添加五块5G的硬盘
SCSI > 创建新虚拟磁盘 > 指定磁盘容量 ,立即分配所有磁盘空间,将虚拟磁盘存储为单个文件
-
修改磁盘属性:
-
-
在mds002主机中:
-
添加5块5G的硬盘
SCSI > 使用已有的虚拟磁盘 > 指定磁盘容量 ,立即分配所有磁盘空间,将虚拟磁盘存储为单个文件 > 选择mds001主机新添加的磁盘文件
-
修改磁盘属性:
-
-
在mds001和mds002的虚拟机目录下,找个后缀名为vmx的文件,在文件末尾添加一下内容:
scsi1.sharedBus = "virtual" disk.locking = "false" diskLib.dataCacheMaxSize = "0" diskLib.dataCacheMaxReadAheadSize = "0" diskLib.dataCacheMinReadAheadSize = "0" diskLib.dataCachePageSize = "4096" diskLib.maxUnsyncedWrites = "0" disk.EnableUUID = "TRUE"
3.完整的脚本:
#! /bin/bash
# 高可用双机GPFS集群的的自动化部署脚本,(部署条件:配置了共享磁盘,网卡已经配置了IP地址,上传gpfs软件包到某个目录下,确保自己内核是3.10.0-1160.el7.x86_64版本)
# 作者:屈**
# 时间:2023年9月22日
# 版本:1.0.0
host_address=(192.168.10.101 192.168.10.102)
host_hostname=(node1 node2)
host_passwd=110119
# NSD磁盘的名称
nsd_name=(NSD_1 NSD_2)
# 输出nsd名字列表以逗号隔开
hostname_douhao=`printf "%s," "${my_array[@]}" | sed 's/,$//'`
# 输出nsd名字列表以分号隔开
hostname_fenhao=`printf "%s," "${my_array[@]}" | sed 's/,$//'`
# NSD磁盘的设备磁盘名在/dev下,可通过lsblk命令查看
nsd_device=(sdb sdc)
# 集群的名字和文件系统的名字
gpfs_cluster_name=gpfs
file_system_name=gpfs1
# 安装expect命令
expect -v &> /dev/null
if [ `echo $?` -ne 0 ];then
echo "没有expect,安装expect命令"
yum install -y expect
fi
# 配置免密登录
echo "########################## 本地开始配置ssh ##########################"
if [ `test -a ~/.ssh/id_rsa.pub;echo $?` == 0 ];then
echo "ssh公钥已创建"
else
echo "ssh公钥未创建,开始创建"
/usr/bin/expect << eof
# 设置捕获字符串后,期待回复的超时时间
set timeout 10
spawn ssh-keygen -t rsa -b 1024
## 开始进连续捕获
expect {
"connecting (yes/no)?" { send "yes\n"; exp_continue }
"s password:" { send "${host_passwd}\n"; exp_continue }
".ssh/id_rsa)" { send "\n"; exp_continue }
"Overwrite (y/n)?" { send "y\n"; exp_continue }
"no passphrase):" { send "\n"; exp_continue }
"passphrase again:" { send "\n"; exp_continue }
}
eof
fi
# 本地的密钥开始加入被控制主机
for ((j=0;j<2;j++));do
echo "########################## ${host_address[j]}正在被添加公钥 ##########################"
/usr/bin/expect << eof
# 设置捕获字符串后,期待回复的超时时间
set timeout 10
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@${host_address[j]}
## 开始进连续捕获
expect {
"connecting (yes/no)?" { send "yes\n"; exp_continue }
"s password:" { send "${host_passwd}\n"; exp_continue }
}
eof
echo "############# ${host_address[j]}配置完毕 #############"
done
# 被控制主机开始创建密钥
for ((j=0;j<2;j++));do
echo "########################## ${host_address[j]}开始创建密钥 ##########################"
if [ `ssh root@${host_address[j]} 'test -a ~/.ssh/id_rsa.pub;echo $?'` == 0 ];then
echo "ssh公钥已创建"
else
echo "ssh公钥未创建,开始创建"
/usr/bin/expect << eof
# 设置捕获字符串后,期待回复的超时时间
set timeout 10
spawn ssh root@${host_address[j]} "ssh-keygen -t rsa -b 1024"
## 开始进连续捕获
expect {
"connecting (yes/no)?" { send "yes\n"; exp_continue }
"s password:" { send "${host_passwd}\n"; exp_continue }
".ssh/id_rsa)" { send "\n"; exp_continue }
"Overwrite (y/n)?" { send "y\n"; exp_continue }
"no passphrase):" { send "\n"; exp_continue }
"passphrase again:" { send "\n"; exp_continue }
}
eof
fi
echo "############# ${host_address[j]}配置完毕 #############"
done
# 被控制主机开始分配密钥
for ((j=0;j<2;j++));do
for((k=0;k<2;k++));do
echo "########################## ${host_address[j]}开始分配公钥给${host_address[k]} ##########################"
/usr/bin/expect << eof
# 设置捕获字符串后,期待回复的超时时间
set timeout 10
spawn ssh -t root@${host_address[j]} "ssh root@${host_address[k]}"
spawn ssh -t root@${host_address[j]} "ssh-copy-id -i /root/.ssh/id_rsa.pub root@${host_address[k]}"
## 开始进连续捕获
expect {
"connecting (yes/no)?" { send "yes\n"; exp_continue }
"s password:" { send "${host_passwd}\n"; exp_continue }
}
eof
echo "############# ${host_address[j]}配置完毕 #############"
done
done
# 修改主机名和配置域名映射
for ((i=0;i<100;i++));do
read -p "修改主机名和配置域名映射?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
echo "########################## 开始配置主机名和域名映射 ##########################"
for ((j=0;j<2;j++));do
if [ `ssh root@${host_address[j]} "hostname"` != "${host_hostname[j]}" ];then
ssh root@${host_address[j]} "hostnamectl set-hostname ${host_hostname[j]}"
fi
ssh root@${host_address[j]} "cat << eof > /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
eof"
for ((k=0;k<2;k++));do
ssh root@${host_address[j]} "echo '${host_address[k]} ${host_hostname[k]}' >> /etc/hosts"
done
done
echo "############# 配置完毕 #############"
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-修改主机名和配置域名映射 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 测试被控主机ssh连接
for ((j=0;j<2;j++));do
echo "########################## ${host_address[j]}开始测试 ##########################"
for ((k=0;k<2;k++));do
/usr/bin/expect << eof
# 设置捕获字符串后,期待回复的超时时间
set timeout 10
spawn ssh -t root@${host_address[j]} "ssh root@${host_address[k]}"
## 开始进连续捕获
expect {
"connecting (yes/no)?" { send "yes\n"; exp_continue }
}
eof
if [ `echo $?` != 0 ];then
echo "${host_hostname[j]}主机无法免密登录${host_hostname[k]}"
exit
fi
done
echo "############# ${host_address[j]}测试完毕 #############"
done
# 配置防火墙和selinux
for ((i=0;i<100;i++));do
read -p "配置防火墙和selinux?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
for ((j=0;j<2;j++));do
echo "########################## ${host_address[j]}开始配置防火墙和selinux ##########################"
ssh root@${host_address[j]} "systemctl stop firewalld;systemctl disable firewalld"
ssh root@${host_address[j]} "sed -i 's/SELINUX=.*/SELINUX=disabled/' /etc/selinux/config"
echo "############# 配置完毕 #############"
done
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-配置防火墙和selinux #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 配置yum源
for ((i=0;i<100;i++));do
read -p "配置yum源?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
echo "########################## 开始配置ssh ##########################"
for ((j=0;j<2;j++));do
echo "########################## 配置${host_address[j]}的本地yum源 ##########################"
ssh root@${host_address[j]} "mkdir /mnt/cdrom &> /dev/null;mount /dev/cdrom /mnt/cdrom"
if [ -z "`ssh root@${host_address[j]} "grep '^\/dev\/cdrom' /etc/fstab"`" ];then
ssh root@${host_address[j]} "cat << eof >> /etc/fstab
/dev/cdrom /mnt/cdrom iso9660 defaults 0 0
eof"
fi
ssh root@${host_address[j]} "cat << eof > /etc/yum.repos.d/centos-local.repo
[centos7.9]
name=centos7.9
baseurl=file:///mnt/cdrom
enabled=1
gpgcheck=0
eof"
echo "############# ${host_address[j]}配置完毕 #############"
echo "########################## 配置${host_address[j]}的扩展源 ##########################"
ssh root@${host_address[j]} "yum install epel-release -y"
echo "############# ${host_address[j]}配置完毕 #############"
echo "########################## 配置${host_address[j]}的阿里yum源 ##########################"
ssh root@${host_address[j]} "yum install -y wget"
if [ `ssh root@${host_address[j]} 'test -a /etc/yum.repos.d/CentOS-Base.repo;echo $?'` == 0 ];then
ssh root@${host_address[j]} "wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo"
fi
ssh root@${host_address[j]} "yum clean all && yum repolist"
echo "############# ${host_address[j]}配置完毕 #############"
if [ `ssh root@${host_address[j]} "echo $?"` != 0 ];then
echo "yum源配置有误,退出执行脚本"
exit
fi
done
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-配置yum源 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 配置chrony时间服务器
for ((i=0;i<100;i++));do
read -p "配置chrony时间服务器?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
echo "########################## 开始配置chrony ##########################"
for ((j=0;j<2;j++));do
if [ `ssh root@${host_address[j]} "systemctl restart chronyd;echo $?"` != 0 ];then
echo "${host_address[j]} 安装chrony"
ssh root@${host_address[j]} "yum install -y chrony && systemctl restart chronyd"
if [ `echo $?` != 0 ];then
echo "安装失败,请排错!"
exit
fi
fi
echo "${host_address[j]}配置chrony"
ssh root@${host_address[j]} "sed -i '/^server/d' /etc/chrony.conf"
if [ ${host_address[j]} == ${host_address[0]} ];then
ssh root@${host_address[j]} "sed -i '2a\server '"${host_address[0]}"' iburst\' /etc/chrony.conf"
ssh root@${host_address[j]} "sed -i 's/#allow 192.168.0.0\/16/allow 192.168.10.0\/16/' /etc/chrony.conf"
ssh root@${host_address[j]} "sed -i 's/#local stratum 10/local stratum 10/' /etc/chrony.conf"
sleep 2
else
ssh root@${host_address[j]} "sed -i '2a\server '"${host_address[0]}"' iburst\' /etc/chrony.conf"
fi
ssh root@${host_address[j]} "systemctl restart chronyd && systemctl enable chronyd &> /dev/null"
sleep 5
ssh root@${host_address[j]} "timedatectl set-ntp true && chronyc sources -v | sed -n '/^\^\*/p'"
if [ -z "`ssh root@${host_address[j]} "chronyc sources -v | sed -n '/^\^\*/p'"`" ];then
echo -e "\e[31m此节点${host_address[j]}的chrony配置有误,请手动调试\e[0m"
exit
fi
echo "############# ${host_address[j]}配置完毕 #############"
done
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-配置chrony时间服务器 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 安装GPFS软件
for ((i=0;i<100;i++));do
read -p "是否安装gpfs软件?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
read -p "请输入你上传到本机的gpfs的(父目录绝对路径)/gpfs*.rpm: " install_path
for ((j=0;j<2;j++));do
echo "########################## ${host_address[j]}开始卸载lustre ##########################"
ssh root@${host_address[j]} "rpm -e gpfs.gskit-8.0.50-86.x86_64"
ssh root@${host_address[j]} "rpm -e gpfs.docs-4.2.3-22.noarch"
ssh root@${host_address[j]} "rpm -e gpfs.msg.en_US-4.2.3-22.noarch"
ssh root@${host_address[j]} "rpm -e gpfs.ext-4.2.3-22.x86_64"
ssh root@${host_address[j]} "rpm -e gpfs.gpl-4.2.3-22.noarch"
ssh root@${host_address[j]} "rpm -e gpfs.base-4.2.3-22.x86_64"
echo "############# ${host_address[j]}卸载完毕 #############"
echo "########################## ${host_address[j]}rpm开始被复制过来 ##########################"
ssh root@${host_address[j]} "rm -rf ~/gpfs-4.2.3-22 && mkdir ~/gpfs-4.2.3-22"
scp ${install_path}/gpfs*.rpm root@${host_address[j]}:~/gpfs-4.2.3-22
if [ `echo $?` != 0 ];then
echo "复制失败,请手动排错!"
exit
fi
echo "############# ${host_address[j]}复制完毕 #############"
echo "########################## ${host_address[j]}开始安装依赖 ##########################"
ssh root@${host_address[j]} "yum -y install make perl rsh ld-linux.so libm.so.6 libc.so.6 ksh libstdc++.so.5 rsh-server rpcbind xinetd libaio cpp gcc-c++ gcc nfs-utils kernel-headers compat-libstdc++ glibc-devel libXp.so.6 imake rpm-build rpm-build m4 python3"
ssh root@${host_address[j]} "yum install -y bash-completion vim net-tools tree psmisc lrzsz dos2unix"
ssh root@${host_address[j]} "yum install iproute -y"
if [ `echo $?` != 0 ];then
echo "安装失败,请手动排错!"
exit
fi
echo "############# ${host_address[j]}安装完毕 #############"
echo "########################## ${host_address[j]}开始安装gpfs ##########################"
ssh root@${host_address[j]} "rpm -ivh ~/gpfs-4.2.3-22/gpfs.base-4.2.3-22.x86_64.rpm"
ssh root@${host_address[j]} "rpm -ivh ~/gpfs-4.2.3-22/gpfs.docs-4.2.3-22.noarch.rpm"
ssh root@${host_address[j]} "rpm -ivh ~/gpfs-4.2.3-22/gpfs.gpl-4.2.3-22.noarch.rpm"
ssh root@${host_address[j]} "rpm -ivh ~/gpfs-4.2.3-22/gpfs.gskit-8.0.50-86.x86_64.rpm"
ssh root@${host_address[j]} "rpm -ivh ~/gpfs-4.2.3-22/gpfs.msg.en_US-4.2.3-22.noarch.rpm"
ssh root@${host_address[j]} "rpm -ivh ~/gpfs-4.2.3-22/gpfs.ext-4.2.3-22.x86_64.rpm"
if [ `echo $?` != 0 ];then
echo "安装失败,请手动排错!"
exit
fi
echo "############# ${host_address[j]}配置完毕 #############"
done
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-安装GPFS软件 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 编译gpfs
for ((i=0;i<100;i++));do
read -p "是否编译GPFS?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
for ((j=0;j<2;j++));do
echo "########################## ${host_address[j]}开始处理编译问题 ##########################"
if [ -z `ssh root@${host_address[j]} "rpm -qa | grep kernel" | grep "kernel-devel-3.10.0-1160.el7.x86_64"` ];then
ssh root@${host_address[j]} "wget http://mirror.centos.org/centos/7/os/x86_64/Packages/kernel-devel-3.10.0-1160.el7.x86_64.rpm"
ssh root@${host_address[j]} "rpm -Uvh kernel-devel-3.10.0-1160.el7.x86_64.rpm --force"
fi
ssh root@${host_address[j]} "sed -i '122s/*/unsigned long __x86_return_thunk;/' /usr/lpp/mmfs/src/gpl-linux/kdump.c"
echo "############# ${host_address[j]}处理完毕 #############"
echo "########################## ${host_address[j]}开始编译gpfs ##########################"
ssh root@${host_address[j]} "cd /usr/lpp/mmfs/src && make Autoconfig LINUX_DISTRIBUTION=REDHAT_AS_LINUX && make World"
if [ `echo $?` != 0 ];then
echo "编译失败,请手动排错!"
exit
fi
ssh root@${host_address[j]} "cd /usr/lpp/mmfs/src && make rpm" | grep "^Wrote:" > ~/rpm.test
gplbin_path=`grep "Wrote" ~/rpm.test | awk -F ': ' '{print $2}'`
ssh root@${host_address[j]} "rpm -Uvh ${gplbin_path} --force"
if [ `echo $?` != 0 ];then
echo "编译失败,请手动排错!"
exit
fi
echo "############# ${host_address[j]}编译完成 #############"
ssh root@${host_address[j]} "source /etc/profile;mmcrcluster"
if [ `echo $?` != 1 ];then
echo "########################## ${host_address[j]}配置环境变量 ##########################"
ssh root@${host_address[j]} "echo 'export PATH=$PATH:/usr/lpp/mmfs/bin' >> /etc/profile"
sleep 3
ssh root@${host_address[j]} "source /etc/profile"
echo "############# ${host_address[j]}环境变量配置完毕 #############"
fi
echo "############# ${host_address[j]}配置完毕 #############"
done
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-编译GPFS #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 创建集群并启动
for ((i=0;i<100;i++));do
read -p "是否创建集群并启动?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
echo "########################## 开始创建集群 ##########################"
if [ -z "`ssh root@${host_address[0]} "source /etc/profile && mmlscluster" | grep 'GPFS cluster name:' | grep "${gpfs_cluster_name}"`" ];then
ssh root@${host_address[0]} "echo -n > ~/gpfsfile"
for ((j=0;j<2;j++));do
ssh root@${host_address[0]} "echo '${host_address[j]}:quorum-manager' >> ~/gpfsfile"
done
ssh root@${host_address[0]} "source /etc/profile && mmcrcluster -N ~/gpfsfile -p ${host_hostname[0]} -s ${host_hostname[1]} -r /usr/bin/ssh -R /usr/bin/scp -A -C ${gpfs_cluster_name}"
if [ `echo $?` != 0 ];then
echo "失败,请手动排错!"
exit
fi
else
echo "########################## 名为gpfs的集群已创建 ##########################"
fi
echo "#################################################################"
echo "########################## 查看集群节点 ##########################"
ssh root@${host_address[0]} "source /etc/profile && mmlscluster"
echo "#################################################################"
echo "#################################################################"
echo "########################## 启动集群并查看集群状态 ##########################"
ssh root@${host_address[0]} "source /etc/profile && mmstartup -a && mmgetstate -Lsa"
if [ `echo $?` != 0 ];then
echo "失败,请手动排错!"
exit
fi
echo "#################################################################"
echo "#################################################################"
echo "############# 配置完毕 #############"
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-创建集群并启动 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 创建NSD磁盘
for ((i=0;i<100;i++));do
read -p "是否创建NSD磁盘?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
echo "########################## 开始创建NSD磁盘 ##########################"
if [ -z "`ssh root@${host_address[0]} "source /etc/profile && mmlsnsd -m" | grep -E '(NSD_1|NSD_2)'`" ];then
ssh root@${host_address[0]} "echo -n > ~/nsd.node"
for ((j=0;j<${#nsd_name[*]};j++));do
ssh root@${host_address[0]} "cat << eof >> ~/nsd.node
%nsd: nsd=${nsd_name[j]}
device=/dev/${nsd_device[j]}
servers=${hostname_douhao}
usage=dataAndMetadata
failureGroup=100
pool=system
eof"
done
ssh root@${host_address[0]} "source /etc/profile && mmcrnsd -F ~/nsd.node -v no"
if [ `echo $?` != 0 ];then
echo "失败,请手动排错!"
exit
fi
else
echo "########################## NSD磁盘已创建 ##########################"
fi
echo "#################################################################"
echo "########################## 查看NSD磁盘 ##########################"
ssh root@${host_address[0]} "source /etc/profile && mmlsnsd -m"
echo "#################################################################"
echo "#################################################################"
echo "########################## 配置仲裁盘 ##########################"
ssh root@${host_address[0]} "source /etc/profile && mmchconfig tiebreakerDisks='${hostname_fenhao}' && mmlsconfig"
if [ `echo $?` != 0 ];then
echo "失败,请手动排错!"
exit
fi
echo "############# 配置完毕 #############"
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-创建NSD磁盘 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done
# 创建文件系统并且挂载
for ((i=0;i<100;i++));do
read -p "是否创建NSD磁盘?(Y/n): " flag
if [ "${flag}" == "Y" ];then
sleep 3
echo "########################## 开始创建NSD磁盘 ##########################"
if [ -z "`ssh root@${host_address[0]} "source /etc/profile && mmlsconfig" | sed -n '/^File systems/,${p}' | grep "${file_system_name}"`" ];then
ssh root@${host_address[0]} "source /etc/profile && mmcrfs /${file_system_name} /dev/${file_system_name} -F ~/nsd.node -A yes -B 512k -m 1 -M 2 -r 1 -R 2 -n 512 -j cluster -Q yes"
if [ `echo $?` != 0 ];then
echo "失败,请手动排错!"
exit
fi
else
echo "########################## gpfs1文件系统已创建 ##########################"
fi
echo "#################################################################"
echo "########################## 查看NSD磁盘 ##########################"
ssh root@${host_address[0]} "source /etc/profile && mmlsconfig"
echo "#################################################################"
echo "#################################################################"
echo "########################## 挂载文件系统 ##########################"
ssh root@${host_address[0]} "source /etc/profile && mmmount all -a && mmlsmount gpfs1 -L"
if [ `echo $?` != 0 ];then
echo "失败,请手动排错!"
exit
fi
echo "############# 配置完毕 #############"
break
elif [ "${flag}" == "n" ];then
echo "############# 已跳过步骤-创建文件系统并且挂载 #############"
break
elif [ ${i} -eq 99 ];then
echo "############# 已退出 #############"
exit
else continue;fi
done