Prometheus快速入门
环境准备
三台主机,配置好主机名
各自配置好主机名
# hostnamectl set-hostname --static server.cluster.com
...
三台都互相绑定IP与主机名
# vim /etc/hosts
192.168.126.143 server.cluster.com
192.168.126.142 agent.cluster.com
192.168.126.145 grafana.cluster.com
时间同步
yum install ntp
systemctl restart ntpd && systemctl enable ntpd
关闭防火墙,selinux
systemctl stop firewalld && systemctl disable firewalld && iptables -F
setenforce 0
安装prometheus
https://prometheus.io/download/
找到对应版本,安装到服务器上,解压
[root@server ~]# tar xf prometheus-2.5.0.linux-amd64.tar.gz -C /usr/local/
[root@server ~]# mv /usr/local/prometheus-2.5.0.linux-amd64/ /usr/local/prometheus
启动程序,默认端口9090
[root@server ~]# /usr/local/prometheus/prometheus --config.file="/usr/local/prometheus/prometheus.yml" &
[root@server ~]# lsof -i:9090
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
prometheu 7667 root 6u IPv6 44101 0t0 TCP *:websm (LISTEN)
prometheu 7667 root 7u IPv4 44103 0t0 TCP localhost:53290->localhost:websm (ESTABLISHED)
prometheu 7667 root 8u IPv6 44104 0t0 TCP localhost:websm->localhost:53290 (ESTABLISHED)
prometheu 7667 root 9u IPv6 44111 0t0 TCP localhost.localdomain:38972->localhost.localdomain:websm (ESTABLISHED)
prometheu 7667 root 10u IPv6 44112 0t0 TCP localhost.localdomain:websm->localhost.localdomain:38972 (ESTABLISHED)
访问server的ip:9090端口
node_exporter安装与监控
在远程主机上安装Download | Prometheus
node_exporter组件
[root@agent ~]# tar xf node_exporter-0.16.0.linux-amd64.tar.gz -C /usr/local/
[root@agent ~]# mv /usr/local/node_exporter-0.16.0.linux-amd64/ /usr/local/node_exporter
[root@agent ~]# ls /usr/local/node_exporter/ 启动
LICENSE node_exporter NOTICE
[root@agent ~]# nohup /usr/local/node_exporter/node_exporter &
[1] 23330
[root@agent ~]# nohup: 忽略输入并把输出追加到"nohup.out"
通过浏览器,访问查看agent监控项
在server上配置
在主配置文件最后加上下面三行
[root@server ~]# vim /usr/local/prometheus/prometheus.yml
- job_name: 'agent' # 取一个job名称来代表被监控的机器
static_configs:
- targets: ['192.168.126.142:9100'] # 这里改成被监控机器的IP,后面端口接9100
[root@server ~]# lsof -i:9090
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
prometheu 7667 root 6u IPv6 44101 0t0 TCP *:websm (LISTEN)
prometheu 7667 root 7u IPv4 44103 0t0 TCP localhost:53290->localhost:websm (ESTABLISHED)
prometheu 7667 root 8u IPv6 44104 0t0 TCP localhost:websm->localhost:53290 (ESTABLISHED)
prometheu 7667 root 9u IPv6 44111 0t0 TCP localhost:38972->localhost:websm (ESTABLISHED)
prometheu 7667 root 10u IPv6 44112 0t0 TCP localhost:websm->localhost:38972 (ESTABLISHED)
[root@server ~]# pkill prometheus #修改完配置文件 重启服务
[root@server ~]# lsof -i:9090
[root@server ~]# /usr/local/prometheus/prometheus --config.file="/usr/local/prometheus/prometheus.yml" &
[1] 23543
[root@server ~]# level=info ts=2023-05-04T16:00:11.354230207Z caller=main.go:244 msg="Starting Prometheus" version="(version=2.5.0, branch=HEAD, revision=67dc912ac8b24f94a1fc478f352d25179c94ab9b)"
level=info ts=2023-05-04T16:00:11.354445095Z caller=main.go:245 build_context="(go=go1.11.1, user=root@578ab108d0b9, date=20181106-11:40:44)"
level=info ts=2023-05-04T16:00:11.354489086Z caller=main.go:246 host_details="(Linux 3.10.0-957.el7.x86_64 #1 SMP Thu Nov 8 23:39:32 UTC 2018 x86_64 server.cluster.com (none))"
level=info ts=2023-05-04T16:00:11.354525505Z caller=main.go:247 fd_limits="(soft=1024, hard=4096)"
level=info ts=2023-05-04T16:00:11.354554918Z caller=main.go:248 vm_limits="(soft=unlimited, hard=unlimited)"
level=info ts=2023-05-04T16:00:11.358685209Z caller=main.go:562 msg="Starting TSDB ..."
level=info ts=2023-05-04T16:00:11.360215886Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1683079200000 maxt=1683115200000 ulid=01GZKMDBKGHH1161QWFQ6EE464
level=info ts=2023-05-04T16:00:11.360393103Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1683115200000 maxt=1683180000000 ulid=01GZKMDBMZ04FVTP1SR9YQ0B2M
level=info ts=2023-05-04T16:00:11.360468037Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1683201600000 maxt=1683208800000 ulid=01GZKME8A224D3NDWWAK8Q6348
level=info ts=2023-05-04T16:00:11.360616789Z caller=repair.go:35 component=tsdb msg="found healthy block" mint=1683180000000 maxt=1683201600000 ulid=01GZKME8B3NEJPEYB1F74NTAEQ
level=info ts=2023-05-04T16:00:11.390491481Z caller=web.go:399 component=web msg="Start listening for connections" address=0.0.0.0:9090
level=info ts=2023-05-04T16:00:11.625912608Z caller=main.go:572 msg="TSDB started"
level=info ts=2023-05-04T16:00:11.626158258Z caller=main.go:632 msg="Loading configuration file" filename=/usr/local/prometheus/prometheus.yml
level=info ts=2023-05-04T16:00:11.630983558Z caller=main.go:658 msg="Completed loading of configuration file" filename=/usr/local/prometheus/prometheus.yml
level=info ts=2023-05-04T16:00:11.631023506Z caller=main.go:531 msg="Server is ready to receive web requests."
[root@server ~]# lsof -i:9090
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
prometheu 23543 root 12u IPv6 83590 0t0 TCP *:websm (LISTEN)
prometheu 23543 root 13u IPv4 83592 0t0 TCP localhost:53294->localhost:websm (ESTABLISHED)
prometheu 23543 root 14u IPv4 83598 0t0 TCP localhost:53296->localhost:websm (ESTABLISHED)
prometheu 23543 root 15u IPv6 83593 0t0 TCP localhost:websm->localhost:53294 (ESTABLISHED)
prometheu 23543 root 16u IPv6 83599 0t0 TCP localhost:websm->localhost:53296 (ESTABLISHED)
查看服务器监控:
监控远程mysql
被管理机agent上安装mysqld_exporter组件
https://prometheus.io/download/
[root@agent ~]# tar xf mysqld_exporter-0.11.0.linux-amd64.tar.gz -C /usr/local/
[root@agent ~]# mv /usr/local/mysqld_exporter-0.11.0.linux-amd64/ /usr/local/mysqld_exporter
[root@agent ~]# ls /usr/local/mysqld_exporter/
LICENSE mysqld_exporter NOTICE
安装mariadb数据库,并授权
[root@agent ~]# yum install mariadb-server -y
systemctl restart mariadb && systemctl enable mariadb
授权
[root@agent ~]# mysql
Welcome to the MariaDB monitor. Commands end with ; or \g.
Your MariaDB connection id is 3
Server version: 5.5.68-MariaDB MariaDB Server
Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others.
Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
MariaDB [(none)]> grant select,replication client,process ON *.* to 'mysql_monitor'@'localhost' identified by '123';
Query OK, 0 rows affected (0.00 sec)
MariaDB [(none)]> flush privileges;
Query OK, 0 rows affected (0.00 sec)
MariaDB [(none)]> quit
Bye
配置文件,写上连接的用户名与密码(和上面的授权的用户名和密码要对应)
[root@agent ~]# vim /usr/local/mysqld_exporter/.my.cnf
[client]
user=mysql_monitor
password=123
[root@agent ~]# nohup /usr/local/mysqld_exporter/mysqld_exporter --config.my-cnf=/usr/local/mysqld_exporter/.my.cnf &
[1] 24167
[root@agent ~]# nohup: 忽略输入并把输出追加到"nohup.out"
[root@agent ~]# lsof -i:9104
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
mysqld_ex 24167 root 3u IPv6 89261 0t0 TCP *:peerwire (LISTEN)
去server配置
在主配置文件最后再加上下面三行
[root@server ~]# vim /usr/local/prometheus/prometheus.yml
- job_name: 'agent_mariadb' # 取一个job名称来代表被监控的mariadb
static_configs:
- targets: ['192.168.126.142:9104'] # 这里改成被监控机器的IP,后面端口接9104
重启服务
[root@server ~]# pkill prometheus
[root@server ~]# lsof -i:9090
[root@server ~]# /usr/local/prometheus/prometheus --config.file="/usr/local/prometheus/prometheus.yml" &
[root@server ~]# lsof -i:9090
进入web界面,点击target
grafana
官网
https://grafana.com/
下载
https://grafana.com/grafana/download
选择另一台主机安装
#根据你虚拟机的型号在官网下载中选择
sudo yum install -y https://dl.grafana.com/enterprise/release/grafana-enterprise-9.5.1-1.x86_64.rpm
[root@grafana ~]# systemctl start grafana-server
[root@grafana ~]# systemctl enable grafana-server
Created symlink from /etc/systemd/system/multi-user.target.wants/grafana-server.service to /usr/lib/systemd/system/grafana-server.service.
#监听端口
[root@grafana ~]# lsof -i:3000
COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME
grafana 7555 grafana 9u IPv6 43695 0t0 TCP *:hbci (LISTEN)
输入IP地址:3000即可访问
(如果无法打开请检查防火墙或者selinux)
首次登录用户名和密码都是admin
并且可以修改
进入主界面
grafana添加数据源
导入mysql监控json模板
https://github.com/percona/grafana-dashboards
-
导入dashboards里的json模板
-
查看错误
-
更改数据源名称
-
再次查看面板
onealert报警
-
为监控项面板创建规则
-
进入睿象云,按照提示配置
睿象云-智能告警平台 (aiops.com)