一、准备环境
1.1 部署服务器
在centos7.9系统里搭建v1.23版本的k8s集群，准备四台服务器，两台作为master，主机名分别为 k8s-master和k8s-master-2，主机名为k8s-master，两台作为 node，主机名分别为k8s-node-1和k8s-node-2
1.2 初始化
对master节点服务器进行环境初始化，包括设置静态IP、修改主机名、关闭firewalld功能以及禁止selinux功能等
执行以下初始化脚本
#中途出现错误，直接退出，不会进行后面的步骤
set -e
#!/bin/bash
#第1步：下载阿里云的centos-7.reop文件
cd /etc/yum.repos.d
curl -O http://mirrors.aliyun.com/repo/Centos-7.repo
#新建存放centos官方提供的repo文件，因为官方不提供服务了
mkdir backup
mv CentOS-* backup
yum makecache
#第2步：修改主机名
hostnamectl set-hostname $1
#第3步：配置静态ip地址
cat >/etc/sysconfig/network-scripts/ifcfg-ens33 <<EOF
BOOTPROTO="none"
NAME="ens33"
DEVICE="ens33"
ONBOOT="yes"
IPADDR=$2
PREFIX=24
GATEWAY=$3
DNS1=114.114.114.114
DNS2=222.246.129.80
EOF
#启动网络服务
service network restart
#第4步： 关闭selinux和firewalld防火墙服务
systemctl stop firewalld
systemctl disable firewalld
#修改/etc/selinux/config文件里的enforcing为disabled
sed -i '/SELINUX=/ s/enforcing/disabled/' /etc/selinux/config
#第5步：关闭交换分区
# 永久关闭
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
第6步：调整内核参数
# 修改linux的内核参数，添加网桥过滤和地址转发功能，转发IPv4并让iptables看到桥接流量
cat <<EOF | tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
# 加载网桥过滤模块
modprobe overlay
modprobe br_netfilter
# 编辑/etc/sysctl.d/kubernetes.conf文件，添加如下配置:
cat << EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
# 应用sysctl参数而不重新启动
sysctl -p
第7步：更新和配置所需源
yum install -y yum-utils device-mapper-persistent-data lvm2 wget net-tools nfs-utils lrzsz gcc gcc-c++ make cmake libxml2-devel openssl-devel curl curl-devel unzip sudo ntp libaio-devel wget vim ncurses-devel autoconf automake zlib-devel python-devel epel-release openssh-server socat ipvsadm conntrack ntpdate telnet
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
第8步：配置ipvs功能
# 安装ipset和ipvsadm
yum install ipset ipvsadm -y
# 添加需要加载的模块写入脚本文件
cat <<EOF > /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
# 为脚本文件添加执行权限
chmod +x /etc/sysconfig/modules/ipvs.modules
# 执行脚本文件
/bin/bash /etc/sysconfig/modules/ipvs.modules
第9步：配置时间同步
systemctl start chronyd && systemctl enable chronyd
第10步：重启
reboot
二、配置docker环境
2.1 安装docker环境
yum install -y docker-ce docker-ce-cli
2.2 配置docker的镜像加速
mkdir -p /etc/docker
cd /etc/docker
vim daemon.json
{
"registry-mirrors": ["https://hub.docker-alhk.dkdun.com/"],
"exec-opts": ["native.cgroupdriver=systemd"]
}
#重新加载docker的配置文件和重启docker服务
systemctl daemon-reload
systemctl restart docker
2.3 设置docker服务开机自启
# 启动docker并设置开机自启
systemctl enable --now docker
# 验证
systemctl status docker
配置cri-docker
kubernets 1.24版本后默认使用containerd做底层容器，需要使用cri-dockerd做中间层来与docker通信
# 下载
wget https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.8/cri-dockerd-0.3.8-3.el7.x86_64.rpm
# 安装
rpm -ivh cri-dockerd-0.3.8-3.el7.x86_64.rpm
# 重载系统守护进程
systemctl daemon-reload
# 修改配置文件
vim /usr/lib/systemd/system/cri-docker.service
# 修改第10行 ExecStart
# 改为
# ExecStart=/usr/bin/cri-dockerd --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9 --container-runtime-endpoint fd://
配置cri-docker服务自启动
# 重载系统守护进程
systemctl daemon-reload
# 启动cri-dockerd
systemctl start cri-docker.socket cri-docker
# 设置cri-dockerd自启动
systemctl enable cri-docker.socket cri-docker
# 检查Docker组件状态
systemctl status docker cir-docker.socket cri-docker
此时，基于这台服务器，克隆另外三台，重新配置IP地址以及主机名
修改每台服务器上的/etc/hosts文件，添加如下配置
vim /etc/hosts
192.168.178.150 k8s-master
192.168.178.151 k8s-node-1
192.168.178.152 k8s-node-2
192.168.178.153 k8s-master-2
接着进行测试，通过访问域名是否可以ping通其他的机器
三、配置k8s集群环境
3.1 配置k8s组件源（需要在每台节点服务器执行）
cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
yum makecache
安装kubectl、kubeadm、kubelet
- kubeadm 是用来安装master节点上的组件（apiserver、etcd、scheduler等），部署k8s集群的。
- kubelet 是用来对容器运行时软件进行管理的（管理docker的）
- kubectl 是用来输入k8s的命令的
yum install -y install kubeadm kubelet kubectl --disableexcludes=kubernetes
设置kubelet自启动
systemctl enable --now kubelet
集群初始化（仅在master节点执行）
init
kubeadm init --kubernetes-version=v1.28.2 \
--pod-network-cidr=10.224.0.0/16 \
--apiserver-advertise-address="此处改为自己master节点服务器的IP地址" \
--image-repository=registry.aliyuncs.com/google_containers \
--cri-socket=unix:///var/run/cri-dockerd.sock
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
查看加入节点的命令
kubeadm token create --print-join-command
kubeadm join 192.168.178.150:6443 --token 6c6k69.vhznczps1pr494sh \
--discovery-token-ca-cert-hash sha256:d095843eb9102f86df65d8a14b20c51aa636eb3429f0252926a51914171a7a95
node节点加入集群
加入work节点
分别在两台work节点服务器执行以下命令
kubeadm join 192.168.178.150:6443 --token 6c6k69.vhznczps1pr494sh \
--discovery-token-ca-cert-hash sha256:d095843eb9102f86df65d8a14b20c51aa636eb3429f0252926a51914171a7a95 \
--cri-socket unix:///var/run/cri-dockerd.sock
分配work
# 在master上执行
kubectl label node k8s-node-1 node-role.kubernetes.io/worker=worker
kubectl label node k8s-node-2 node-role.kubernetes.io/worker=worker
加入master节点
在k8s-master-2创建证书存放目录：
cd /root && mkdir -p /etc/kubernetes/pki/etcd &&mkdir -p ~/.kube/
把k8s-master节点的证书拷贝到k8s-master-2上（已经搭建了免密通道可以直接远程复制，如果没有，输入密码即可）：
scp /etc/kubernetes/pki/ca.crt k8s-master-2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/ca.key k8s-master-2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.key k8s-master-2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.pub k8s-master-2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.crt k8s-master-2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.key k8s-master-2:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/etcd/ca.crt k8s-master-2:/etc/kubernetes/pki/etcd/
scp /etc/kubernetes/pki/etcd/ca.key k8s-master-2:/etc/kubernetes/pki/etcd/
在master-k8s服务器检查 kubeadm-config ConfigMap 是否正确配置了 controlPlaneEndpoint。可以使用 kubectl 命令获取 kubeadm-config ConfigMap 的信息：
kubectl -n kube-system edit cm kubeadm-config -o yaml
添加如下字段：controlPlaneEndpoint: “192.168.178.150:6443” （k8s-master节点服务器的IP地址）
apiVersion: v1
data:
ClusterConfiguration: |
apiServer:
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: "192.168.178.150:6443"
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.28.2
networking:
dnsDomain: cluster.local
podSubnet: 10.224.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
kind: ConfigMap
metadata:
creationTimestamp: "2024-08-16T08:13:08Z"
name: kubeadm-config
namespace: kube-system
resourceVersion: "3713"
uid: 7d5825b4-3e25-4269-a61c-833cbdd19770
重启kubelet
systemctl restart kubelet
在k8s-master节点服务器查看加入节点的命令：
kubeadm token create --print-join-command
kubeadm join 192.168.178.150:6443 --token myfuwt.po8c3fasv56k1z3d \
--discovery-token-ca-cert-hash sha256:d095843eb9102f86df65d8a14b20c51aa636eb3429f0252926a51914171a7a95
将k8s-master-2以master的身份加入
kubeadm join 192.168.178.150:6443 --token myfuwt.po8c3fasv56k1z3d \
--discovery-token-ca-cert-hash sha256:d095843eb9102f86df65d8a14b20c51aa636eb3429f0252926a51914171a7a95 \
--cri-socket unix:///var/run/cri-dockerd.sock \
--control-plane --ignore-preflight-errors=SystemVerification
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
在k8s-master/k8s-master-2上查看集群状况：
kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master NotReady control-plane 48m v1.28.2
k8s-master-2 NotReady control-plane 3m36s v1.28.2
k8s-node-1 NotReady <none> 44m v1.28.2
k8s-node-2 NotReady <none> 44m v1.28.2
安装Calico网络插件
在master执行
kubectl apply -f https://docs.projectcalico.org/archive/v3.25/manifests/calico.yaml
验证，查看状态
kubectl get node
NAME STATUS ROLES AGE VERSION
k8s-master Ready control-plane 23h v1.28.2
k8s-master-2 Ready control-plane 22h v1.28.2
k8s-node-1 Ready worker 23h v1.28.2
k8s-node-2 Ready worker 23h v1.28.2
k8s配置ipvs
ipvs 是linux里的一个负载均衡软件，默认在linux内核里就安装了，修改k8s里的一个配置，负载均衡的时候使用ipvs做为默认的负载均衡软件，如果不修改默认是iptables
kubectl edit configmap kube-proxy -n kube-system
# 修改配置
mode: "ipvs"
# 删除所有kube-proxy pod使之重启
kubectl delete pods -n kube-system -l k8s-app=kube-proxy
安装Dashboard（只在master节点上执行）
下载安装
wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml
修改Service部分，改为NodePort对外暴露端口
安装
kubectl apply -f recommended.yaml
查看安装效果
kubectl get pods,svc -n kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
pod/dashboard-metrics-scraper-5657497c4c-5vcpn 1/1 Running 0 3m58s
pod/kubernetes-dashboard-78f87ddfc-ccwdm 1/1 Running 0 3m58s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/dashboard-metrics-scraper ClusterIP 10.96.38.237 <none> 8000/TCP 3m58s
service/kubernetes-dashboard NodePort 10.107.43.228 <none> 443:30503/TCP 3m58s
创建账号
创建dashboard-access-token.yaml文件
vim dashboard-access-token.yaml
# Creating a Service Account
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
---
# Creating a ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
---
# Getting a long-lived Bearer Token for ServiceAccount
apiVersion: v1
kind: Secret
metadata:
name: admin-user
namespace: kubernetes-dashboard
annotations:
kubernetes.io/service-account.name: "admin-user"
type: kubernetes.io/service-account-token
# Clean up and next steps
# kubectl -n kubernetes-dashboard delete serviceaccount admin-user
# kubectl -n kubernetes-dashboard delete clusterrolebinding admin-user
执行
kubectl apply -f dashboard-access-token.yaml
获取token
kubectl get secret admin-user -n kubernetes-dashboard -o jsonpath={".data.token"} | base64 -d
访问dashboard
kubectl get svc -n kubernetes-dashboard
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
dashboard-metrics-scraper ClusterIP 10.96.38.237 <none> 8000/TCP 9m39s
kubernetes-dashboard NodePort 10.107.43.228 <none> 443:30503/TCP 9m39s
浏览器访问集群ip:端口(https://192.168.178.150:30503/)，注意https，而后输入上一步获取到的token即可
但我发现15分钟不对dashboard进行操作时，token会自动过期
解决token默认15分钟过期的问题
vim recommended.yaml
添加如下图所示的这条配置，超过时间调整为12小时
重新应用
kubectl apply -f recommended.yaml
安装kuboard（任意节点执行，推荐master）
kuboard 是一个多k8s集群的管理软件，假如公司内部有多套k8s集群，可以使用kuboard来进行管理。
sudo docker run -d \
--restart=unless-stopped \
--name=kuboard \
-p 80:80/tcp \ # 可根据需要修改第一个暴露的port
-p 10081:10081/tcp \ # 无特殊需要不建议修改
-e KUBOARD_ENDPOINT="http://192.168.178.150:10081" \ # 部署在哪台机器就用什么IP:PORT
-e KUBOARD_AGENT_SERVER_TCP_PORT="10081" \
-v /kuboard-data:/data \ # 可根据需要修改第一个数据挂载路径
swr.cn-east-2.myhuaweicloud.com/kuboard/kuboard:v3
访问的是master节点的http://192.168.178.150
- 用户名：
admin - 密 码：
Kuboard123
进入dashboard后，将自己的k8s集群导入进去
点击确定，选择集群使用身份，而后查看集群摘要
如下就是成功效果（看到计算资源的效果需要先安装metric-server服务）
安装kubectl自动补全命令
yum install bash-completion -y
永久设置自动补全
echo "source <(kubectl completion bash)" >> ~/.bashrc && bash
部署metric-server（在master执行）
下载
wget https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.6.2/components.yaml
修改components.yaml配置
containers:
- args:
...
- --kubelet-insecure-tls # 添加这一行
image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.6.2 # 修改镜像仓库地址
应用
kubectl apply -f components.yaml
查看效果
kubectl top nodes
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
k8s-master 258m 12% 1881Mi 51%
k8s-master-2 171m 8% 1067Mi 29%
k8s-node-1 60m 3% 702Mi 40%
k8s-node-2 75m 3% 785Mi 45%
安装promethes、grafana、node-exporter
拉取镜像
docker pull prom/node-exporter
docker pull prom/prometheus
docker pull grafana/grafana
安装node-exporter.yaml
cat node-exporter.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: node-exporter
namespace: kube-system
labels:
k8s-app: node-exporter
spec:
selector:
matchLabels:
k8s-app: node-exporter
template:
metadata:
labels:
k8s-app: node-exporter
spec:
containers:
- image: prom/node-exporter
name: node-exporter
ports:
- containerPort: 9100
protocol: TCP
name: http
---
apiVersion: v1
kind: Service
metadata:
labels:
k8s-app: node-exporter
name: node-exporter
namespace: kube-system
spec:
ports:
- name: http
port: 9100
nodePort: 31672
protocol: TCP
type: NodePort
selector:
k8s-app: node-exporter
kubectl apply -f node-exporter.yaml
查看效果
kubectl get daemonset -n kube-system
NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE
calico-node 4 4 4 4 4 kubernetes.io/os=linux 21h
kube-proxy 4 4 4 4 4 kubernetes.io/os=linux 23h
node-exporter 2 2 2 2 2 <none> 96m
kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 23h
metrics-server ClusterIP 10.110.40.26 <none> 443/TCP 5h18m
node-exporter NodePort 10.108.60.168 <none> 9100:31672/TCP 97m
kubectl get pod -A -o wide
NAMESPACE NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
kube-system node-exporter-dqffx 1/1 Running 0 99m 10.224.109.69 k8s-node-1 <none> <none>
kube-system node-exporter-vslwc 1/1 Running 0 99m 10.224.140.72 k8s-node-2 <none> <none>
部署prometheus
给prometheus创建一个集群角色（ClusterRole）和服务账号（SA），然后通过ClusterRoleBinding将SA和ClusterRole绑定
cat rbac-setup.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: prometheus
rules:
- apiGroups: [""]
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs: ["get", "list", "watch"]
- apiGroups:
- extensions
resources:
- ingresses
verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: prometheus
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: prometheus
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: prometheus
subjects:
- kind: ServiceAccount
name: prometheus
namespace: kube-system
kubectl apply -f rbac-setup.yaml
cat configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: prometheus-config
namespace: kube-system
data:
prometheus.yml: |
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: 'kubernetes-apiservers'
kubernetes_sd_configs:
- role: endpoints
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
action: keep
regex: default;kubernetes;https
- job_name: 'kubernetes-nodes'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics
- job_name: 'kubernetes-cadvisor'
kubernetes_sd_configs:
- role: node
scheme: https
tls_config:
ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
relabel_configs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- target_label: __address__
replacement: kubernetes.default.svc:443
- source_labels: [__meta_kubernetes_node_name]
regex: (.+)
target_label: __metrics_path__
replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
- job_name: 'kubernetes-service-endpoints'
kubernetes_sd_configs:
- role: endpoints
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
action: replace
target_label: __scheme__
regex: (https?)
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
action: replace
target_label: __address__
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
action: replace
target_label: kubernetes_name
- job_name: 'kubernetes-services'
kubernetes_sd_configs:
- role: service
metrics_path: /probe
params:
module: [http_2xx]
relabel_configs:
- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__address__]
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_service_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_service_name]
target_label: kubernetes_name
- job_name: 'kubernetes-ingresses'
kubernetes_sd_configs:
- role: ingress
relabel_configs:
- source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
action: keep
regex: true
- source_labels: [__meta_kubernetes_ingress_scheme,__address__,__meta_kubernetes_ingress_path]
regex: (.+);(.+);(.+)
replacement: ${1}://${2}${3}
target_label: __param_target
- target_label: __address__
replacement: blackbox-exporter.example.com:9115
- source_labels: [__param_target]
target_label: instance
- action: labelmap
regex: __meta_kubernetes_ingress_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_ingress_name]
target_label: kubernetes_name
- job_name: 'kubernetes-pods'
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
action: keep
regex: true
- source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
action: replace
target_label: __metrics_path__
regex: (.+)
- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
action: replace
regex: ([^:]+)(?::\d+)?;(\d+)
replacement: $1:$2
target_label: __address__
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- source_labels: [__meta_kubernetes_namespace]
action: replace
target_label: kubernetes_namespace
- source_labels: [__meta_kubernetes_pod_name]
action: replace
target_label: kubernetes_pod_name
kubectl apply -f configmap.yaml
使用部署控制器（deployment）去起一个pod来部署prometheus
cat prometheus.deploy.yml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
name: prometheus-deployment
name: prometheus
namespace: kube-system
spec:
replicas: 1
selector:
matchLabels:
app: prometheus
template:
metadata:
labels:
app: prometheus
spec:
containers:
- image: prom/prometheus
name: prometheus
command:
- "/bin/prometheus"
args:
- "--config.file=/etc/prometheus/prometheus.yml"
- "--storage.tsdb.path=/prometheus"
- "--storage.tsdb.retention=24h"
ports:
- containerPort: 9090
protocol: TCP
volumeMounts:
- mountPath: "/prometheus"
name: data
- mountPath: "/etc/prometheus"
name: config-volume
resources:
requests:
cpu: 100m
memory: 100Mi
limits:
cpu: 500m
memory: 2500Mi
serviceAccountName: prometheus
volumes:
- name: data
emptyDir: {}
- name: config-volume
configMap:
name: prometheus-config
kubectl apply -f prometheus.deploy.yml
启动一个service去发布它
cat prometheus.svc.yml
kind: Service
apiVersion: v1
metadata:
labels:
app: prometheus
name: prometheus
namespace: kube-system
spec:
type: NodePort
ports:
- port: 9090
targetPort: 9090
nodePort: 30003
selector:
app: prometheus
kubectl apply -f prometheus.svc.yml
部署grafana
cat grafana-deploy.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: grafana-core
namespace: kube-system
labels:
app: grafana
component: core
spec:
replicas: 1
selector:
matchLabels:
app: grafana
template:
metadata:
labels:
app: grafana
component: core
spec:
containers:
- image: grafana/grafana
name: grafana-core
imagePullPolicy: IfNotPresent
# env:
resources:
# keep request = limit to keep this container in guaranteed class
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 100m
memory: 100Mi
env:
# The following env variables set up basic auth twith the default admin user and admin password.
- name: GF_AUTH_BASIC_ENABLED
value: "true"
- name: GF_AUTH_ANONYMOUS_ENABLED
value: "false"
# - name: GF_AUTH_ANONYMOUS_ORG_ROLE
# value: Admin
# does not really work, because of template variables in exported dashboards:
# - name: GF_DASHBOARDS_JSON_ENABLED
# value: "true"
readinessProbe:
httpGet:
path: /login
port: 3000
# initialDelaySeconds: 30
# timeoutSeconds: 1
#volumeMounts: #先不进行挂载
#- name: grafana-persistent-storage
# mountPath: /var
#volumes:
#- name: grafana-persistent-storage
#emptyDir: {}
kubectl apply -f grafana-deploy.yaml
cat grafana-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: grafana
namespace: kube-system
labels:
app: grafana
component: core
spec:
type: NodePort
ports:
- port: 3000
targetPort: 3000
nodePort: 30124
selector:
app: grafana
component: core
kubectl apply -f grafana-svc.yaml
cat grafana-ing.yaml
kind: Ingress
metadata:
name: grafana
namespace: kube-system
spec:
rules:
- host: k8s.grafana
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: grafana
port:
number: 3000
kubectl apply -f grafana-ing.yaml
查看效果
查看pod和service
访问http://192.168.178.150:31672/metrics，这就是node-exporter采集的数据
访问prometheus地址http://192.168.178.150:30003/graph
访问grafana的地址http://192.168.178.150:30124/login，默认用户密码都是admin，登录后会默认让你修改密码
登录成功后添加Data sources
添加Dashboards，导入315模板
