前期准备
- 创建用户：
sudo useradd hadoop
sudo passwd hadoop
# 密码
xxx
- 系统层面，禁用内存交换
sudo swapoff -a - 修改
sudo vi /etc/security/limits.conf
hadoop hard memlock unlimited
hadoop soft memlock unlimited
hadoop soft nofile 65536
hadoop hard nofile 65536
hadoop soft nproc 4096
hadoop hard nproc 4096
- 增加系统配置
sudo vi /etc/sysctl.conf
vm.max_map_count = 262144
# 保存，执行
sudo sysctl -p
- 配置hosts
- 切换用户，配置免密
sudo su hadoop
ssh-keygen -t rsa
ssh-copy-id xxx
- 如果是debian系统的话，切换用户后不显示路径，需要改
vi /etc/passwd，将hadoop用户的/bin/sh改为/bin/bash
- 为了操作方便，也可以给hadoop一个root权限
vi /etc/sudoers
hadoop ALL=(ALL) NOPASSWD: ALL
- 为了操作方便，给主节点两个方便管理的脚本
1、sudo vi /bin/xcall
#!/bin/bash
# 获取控制台指令
cmd=$*
# 判断指令是否为空
if [ ! -n "$cmd" ]
then
echo "command can not be null !"
exit
fi
# 获取当前登录用户
user=`whoami`
# 在从机执行指令,这里需要根据你具体的集群情况配置，host与具体主机名一致，同上
for (( ip=71;ip<=80;ip++ ))
do
echo "================current ip is 172.16.75.$ip================="
ssh $user@192.168.10.$ip $cmd
done
2、sudo vi /bin/xsync
#!/bin/bash
# 获取输出参数，如果没有参数则直接返回
pcount=$#
if [ $pcount -eq 0 ]
then
echo "no parameter find !";
exit;
fi
# 获取传输文件名
p1=$1
filename=`basename $p1`
echo "load file $p1 success !"
# 获取文件的绝对路径
pdir=`cd -P $(dirname $p1); pwd`
echo "file path is $pdir"
# 获取当前用户（如果想使用root用户权限拷贝文件，在命令后加入-root参数即可）
user=$2
case "$user" in
"-root")
user="root";;
"")
user=`whoami`;;
*)
echo "illegal parameter $user"
esac
echo $user
# 拷贝文件到从机(这里注意主机的host需要根据你的实际情况配置，要与你具体的主机名对应)
for (( ip=72;ip<=93;ip++ ))
do
echo "================current host is 172.16.87.$ip================="
rsync -rvl $pdir/$filename $user@192.168.10.$ip:$pdir
done
echo "complate !"
3、sudo chown hadoop:hadoop /bin/xcall /bin/xsync
4、sudo chmod u+x /bin/xcall /bin/xsync
ES
- 创建父目录，改权限
sudo mkdir /data/services
sudo chown -R hadoop:hadoop /data/services
- 解压，安装插件
tar zxvf elasticsearch-8.14.3.tar.gz
- 配置jvm.options和elasticsearch.yml
cluster.name: gpt-v8
node.name: node-01
path.data: /data/services/elasticsearch-8.14.3/data
path.logs: /data/services/elasticsearch-8.14.3/logs
network.host: node01
http.port: 9200
discovery.seed_hosts: ["node01", "node02", "node03"]
cluster.initial_master_nodes: ["node01", "node02", "node03"]
# 节点角色
node.roles: [master]
# 是否支持跨域
http.cors.enabled: true
# *表示支持所有域名
http.cors.allow-origin: "*"
xpack.security.enabled: false
- 分发配置，各节点修改配置
- 启动ES：
/data/services/elasticsearch-8.14.3/bin/elasticsearch -d
kibana
- 修改配置，直接启动
server.port: 5601
server.host: "node01"
elasticsearch.hosts: ["http://node01:9200", "http://node02:9200", "http://node03:9200"]
# 启动
nohup /data/services/kibana-8.14.3/bin/kibana >> /data/services/kibana-8.14.3/logs/kibana.log 2>&1 &
安全配置
- 生成证书：
bin/elasticsearch-certutil ca --silent --pem --days 3650 -out config/certs/ca.zip - 解压
cd config/certs/ && unzip ca.zip得到 ca 目录，包含：ca.crt和ca.key - 各个节点生成证书：
vi instance.yml
instances:
- name: node-01
dns:
- localhost
- bigdata-offline-elasticsearch-node01
ip:
- "xxx1"
- name: node-02
dns:
- localhost
- bigdata-offline-elasticsearch-node02
ip:
- "xxx2"
- name: node-03
dns:
- localhost
- bigdata-offline-elasticsearch-node03
ip:
- "xxx3"
- name: "kibana"
ip:
- "xxx3"
- 生成节点证书：
bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key - 解压：
cd config/certs/ && unzip certs.zip，包含：instance.crt和instance.crt - 各个节点配置 elasticsearch.yml：
xpack.security.enabled: true
xpack.monitoring.collection.enabled: true
xpack.security.http.ssl.enabled: false # 启用 HTTPS（建议）
xpack.security.transport.ssl.enabled: true # 节点间加密（必须）
#xpack.security.http.ssl.key: certs/instance.key
#xpack.security.http.ssl.certificate: certs/instance.crt
#xpack.security.http.ssl.certificate_authorities: certs/ca.crt
xpack.security.transport.ssl.key: certs/node-01/node-01.key
xpack.security.transport.ssl.certificate: certs/node-01/node-01.crt
xpack.security.transport.ssl.certificate_authorities: certs/ca/ca.crt
- 挨个重启服务
- 重置密码：
bin/elasticsearch-setup-passwords auto - 验证：
curl -k -u elastic:password http://node-01:9200
{
"name" : "node-01",
"cluster_name" : "gpt-v8",
"cluster_uuid" : "RY5geW7yRPq2YHouM3AQLA",
"version" : {
"number" : "8.14.3",
"build_flavor" : "default",
"build_type" : "tar",
"build_hash" : "d55f984299e0e88dee72ebd8255f7ff130859ad0",
"build_date" : "2024-07-07T22:04:49.882652950Z",
"build_snapshot" : false,
"lucene_version" : "9.10.0",
"minimum_wire_compatibility_version" : "7.17.0",
"minimum_index_compatibility_version" : "7.0.0"
},
"tagline" : "You Know, for Search"
}
配置kibana
- 拷贝证书
- 配置 kibana.yml
elasticsearch.username: "kibana_system"
elasticsearch.password: "password"
elasticsearch.ssl.certificateAuthorities: [ "/data/services/kibana-8.14.3/config/certs/ca/ca.crt" ]
- 重启服务
- 使用elastic用户登录
为某个用户分配token
- 在kibana的Management的Stack Management，Security创建role和user，配置权限
- 为该用户分配token：
POST _security/api_key
{
"name": "similar"
}
# 查询：
GET _security/api_key
参考
