Prometheus v2.2.1
编写yaml文件，包含创建ns、configmap、deployment、service
# 创建monitoring空间
vi prometheus-ns.yaml
apiVersion: v1
kind: Namespace
metadata:
name: monitor-sa
# 创建SA并绑定权限
kubectl create serviceaccount monitor -n monitor-sa
kubectl create clusterrolebinding monitor-clusterrolebinding -n monitor-sa --clusterrole=cluster-admin --serviceaccount=monitor-sa:monitor
kubectl create clusterrolebinding monitor-clusterrolebinding-1 -n monitor-sa --clusterrole=cluster-admin --user=system:serviceaccount:monitor-sa:monitor
# 创建cm、deployment、svc
apiVersion: v1
kind: ConfigMap
metadata:
labels:
app: prometheus
name: prometheus-config
namespace: monitor-sa
data:
prometheus.yml: |
global:
scrape_interval: 15s
scrape_timeout: 10s
evaluation_interval: 1m
scrape_configs:
- job_name: 'kubernetes-node'
kubernetes_sd_configs:
- role: node
relabel_configs:
- source_labels: [__address__]
regex: '(.*):10250'
replacement: '${1}:9100'
target_label: __address__
action: replace
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- job_name: 'kubernetes-node-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-apiserver'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_po
rt_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: prometheus-server
namespace: monitoring
labels:
app: prometheus
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
component: server
#matchExpressions:
#- {key: app, operator: In, values: [prometheus]}
---
apiVersion: v1
kind: Service
metadata:
name: prometheus
namespace: monitor-sa
labels:
app: prometheus
annotations: # 增加注解，可被prometheus监控到
prometheus.io/scrape: "true"
spec:
type: NodePort
ports:
- port: 9090
targetPort: 9090
protocol: TCP
selector:
app: prometheus
component: server
[root@mast01 prometheus]# kubectl get pod -n monitor-sa
NAME READY STATUS RESTARTS AGE
prometheus-dfdfb9d79-h4tk4 1/1 Running 0 22h
[root@mast01 prometheus]# kubectl get svc -n monitor-sa
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
prometheus NodePort 10.109.44.37 <none> 9090:30090/TCP 22h
[root@mast01 prometheus]# curl 10.109.44.37:9090
<a href="/graph">Found</a>.
[root@mast01 prometheus]# kubectl get ep -n monitor-sa
NAME ENDPOINTS AGE
prometheus 10.244.140.67:9090 22h
[root@mast01 prometheus]# curl 172.16.80.131:30090
<a href="/graph">Found</a>.
浏览器访问 172.16.80.131:30090
node-exporter组件安装和配置
node-exporter可以采集机器（物理机、虚拟机、云主机等）的监控指标数据，能够采集到的指标包括CPU, 内存，磁盘，网络，文件数等信息。
# 上传node-exporter.tar.gz到harbor
docker load -i node-exporter.tar.gz
docker tag prom/node-exporter:v0.16.0 172.16.80.140/node-exporter/node-exporter:v0.16.0
docker push 172.16.80.140/node-exporter/node-exporter:v0.16.0
# 创建daemon控制器的yaml
[root@mast01 prometheus]# vi node-export.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: node-exporter
namespace: monitor-sa
labels:
name: node-exporter
spec:
selector:
matchLabels:
name: node-exporter
template:
metadata:
labels:
name: node-exporter
spec:
hostPID: true
hostIPC: true
hostNetwork: true
containers:
- name: node-exporter
image: 172.16.80.140/node-exporter/node-exporter:v0.16.0
imagePullPolicy: IfNotPresent
ports:
- containerPort: 9100
resources:
requests:
cpu: 0.15
securityContext:
privileged: true
args:
- --path.procfs
- /host/proc
- --path.sysfs
- /host/sys
- --collector.filesystem.ignored-mount-points
- '"^/(sys|proc|dev|host|etc)($|/)"'
volumeMounts:
- name: dev
mountPath: /host/dev
- name: proc
mountPath: /host/proc
- name: sys
mountPath: /host/sys
- name: rootfs
mountPath: /rootfs
tolerations:
- key: "node-role.kubernetes.io/control-plane"
operator: "Exists"
effect: "NoSchedule"
volumes:
- name: proc
hostPath:
path: /proc
- name: dev
hostPath:
path: /dev
- name: sys
hostPath:
path: /sys
- name: rootfs
hostPath:
path: /
# hostNetwork、hostIPC、hostPID都为True时，表示这个Pod里的所有容器，会直接使用宿主机的网络，直接与宿主机进行IPC（进程间通信）通信，可以看到宿主机里正在运行的所有进程。加入了hostNetwork:true会直接将我们的宿主机的9100端口映射出来，从而不需要创建service 在我们的宿主机上就会有一个9100的端口
该yaml会在每个节点上部署一个node-exporter
[root@mast01 prometheus]# netstat -an | grep 9100
tcp6 0 0 :::9100 :::* LISTEN
[root@mast01 prometheus]# kubectl get pods -n monitor-sa -owide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
node-exporter-5ms5j 1/1 Running 0 7s 172.16.80.133 node02 <none> <none>
node-exporter-dgg2z 1/1 Running 0 9s 172.16.80.131 mast01 <none> <none>
node-exporter-k4mf5 1/1 Running 0 6s 172.16.80.132 node01 <none> <none>
通过curl可获取主机信息
[root@mast01 prometheus]# curl http://172.16.80.131:9100/metrics | more
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 0.00010856
go_gc_duration_seconds{quantile="0.25"} 0.00010856
go_gc_duration_seconds{quantile="0.5"} 0.000879866
go_gc_duration_seconds{quantile="0.75"} 0.000879866
go_gc_duration_seconds{quantile="1"} 0.000879866
go_gc_duration_seconds_sum 0.000988426
go_gc_duration_seconds_count 2
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 6
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.9.6"} 1
# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use.
# TYPE go_memstats_alloc_bytes gauge
go_memstats_alloc_bytes 1.863448e+06
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 5.928912e+06
......
curl http://172.16.80.131:9100/metrics | grep node_cpu_seconds # 显示cpu使用情况
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 91593 100 91593 0 0 6934k 0 --:--:-- # HELP node_cpu_seconds_total Seconds the cpus spent in each mode.
--:--# TYPE node_cpu_seconds_total counter # counter类型，只增不减
:--node_cpu_seconds_total{cpu="0",mode="idle"} 74516.69
-node_cpu_seconds_total{cpu="0",mode="iowait"} 13.52
-:node_cpu_seconds_total{cpu="0",mode="irq"} 0
--node_cpu_seconds_total{cpu="0",mode="nice"} 0.03
:-node_cpu_seconds_total{cpu="0",mode="softirq"} 76.96
- node_cpu_seconds_total{cpu="0",mode="steal"} 0
74node_cpu_seconds_total{cpu="0",mode="system"} 601.74
53node_cpu_seconds_total{cpu="0",mode="user"} 1234.91
knode_cpu_seconds_total{cpu="1",mode="idle"} 74511.08
node_cpu_seconds_total{cpu="1",mode="iowait"} 24.89
node_cpu_seconds_total{cpu="1",mode="irq"} 0
node_cpu_seconds_total{cpu="1",mode="nice"} 0.18
node_cpu_seconds_total{cpu="1",mode="softirq"} 86.84
node_cpu_seconds_total{cpu="1",mode="steal"} 0
node_cpu_seconds_total{cpu="1",mode="system"} 598.25
node_cpu_seconds_total{cpu="1",mode="user"} 1217.33
curl http://172.16.80.131:9100/metrics | grep node_load # 最近一分钟内负载使用情况
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0# HELP node_load1 1m load average.
# TYPE node_load1 gauge # gauge类型，可增可减
node_load1 0.27
# HELP node_load15 15m load average.
# TYPE node_load15 gauge
node_load15 0.42
# HELP node_load5 5m load average.
# TYPE node_load5 gauge
node_load5 0.43
100 91584 100 91584 0 0 8262k 0 --:--:-- --:--:-- --:--:-- 8943k
通过prometheus，可以看到监控对象
Graph选择对应指标和周期，可以打印出指标图形
prometheus热加载：
svc配置修改后，prometheus并不会马上更新配置，可以通过reload这个svc对应pod的配置，这样比较安全快速的使prometheus进行了热加载，并不会对其他产生影响
# 获取svc对应的pod的ip
kubectl get pods -owide
# 热加载
curl -X POST http://podip:9090/-/reload
如果直接删除prometheus，加载配置，也可以达到更新目的，但这种暴力加载会使监控数据丢失，不建议使用
