1.日志收集场景分析与说明
部署架构说明
对于那些将日志输出到，stdout与stderr的Pod，
可以直接使用DaemonSet控制器在每个Node节点上运行一个 <font color=red>filebeat、logstash、fluentd </font>容器进行统一的收集，而后写入到日志存储系统。
2.filebeat部署
#创建ServiceAccount
kubectl create serviceaccount filebeat -n logging
#创建ClusterRole
kubectl create clusterrole filebeat \
--verb=get,list,watch \
--resource=namespace,pods,nodes
#创建ClusterRolebinding
kubectl create clusterrolebinding filebeat \
--serviceaccount=logging:filebeat \
--clusterrole=filebeat
#下载镜像 ，推送
docker pull docker.elastic.co/beats/filebeat:7.17.6
docker tag docker.elastic.co/beats/filebeat:7.17.6 harbor.oldxu.net/base/filebeat:7.17.6
docker push harbor.oldxu.net/base/filebeat:7.17.6
2.1 交付filebeat
1、从ConfigMap中挂载filebeat.yaml配置文件；
2、挂载 /var/log、/var/lib/docker/containers 日志相关目录；
3、使用 hostPath 方式挂载 /usr/share/filebeat/data 数据目录，该目录下有一个registry文件，里面记录了filebeat采集日志位置的相关
内容，比如文件offset、source、timestamp等，如果Pod发生异常后K8S自动将Pod进行重启，不挂载的情况下registry会被重置，将导致日志文件又从offset=0开始采集，会造成重复收集日志。这点非常重要.
filebeat-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: filebeat
namespace: logging
spec:
selector:
matchLabels:
app: filebeat
template:
metadata:
labels:
app: filebeat
spec:
serviceAccountName: "filebeat"
tolerations:
- key: node-role.kubernetes.io/master
operator: "Exists"
effect: "NoSchedule"
imagePullSecrets:
- name: harbor-admin
containers:
- name: filebeat
image: harbor.oldxu.net/base/filebeat:7.17.6
args: [
"-c","/etc/filebeat.yml",
"-e"
]
securityContext:
runAsUser: 0
resources:
limits:
memory: 200Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
subPath: filebeat.yml
- name: varlog
mountPath: /var/log
readOnly: true
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
volumes:
- name: config
configMap:
name: filebeat-config
- name: varlog
hostPath:
path: /var/log
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
2.2 收集kube-system名称空间
日志位置： /var/log/containers/${pod_name}_${pod_namespace}_${container_name}-${container_id}
filebeat-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
data:
filebeat.yml: |-
#===Filebeat autodiscover
providers:
- type: kubernetes
templates:
- condition: #匹配kube-system名称空间下所有日志
equals:
kubernetes.namespace: kube-system
config:
- type: container
stream: all #收集stdout、stderr类型日志，all是所有
encoding: utf-8
paths: /var/log/containers/*-${data.kubernetes.container.id}.log
exclude_lines: ['info']
#=== Kafka Output
output.console:
pretty: true
enable: true
下面是 正确的filebeat-cm.yaml ，上面留着做错误参考
[root@master01 02-DaemonSet-agent-log]# cat 02-filebeat-cm.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
data:
filebeat.yml: |-
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
- condition:
equals:
kubernetes.namespace: kube-system
config:
- type: container
stream: all
encoding: utf-8
paths: /var/log/containers/*-${data.kubernetes.container.id}.log
exclude_lines: ['info']
output.console:
pretty: true
enable: true
备注：要写对 filebeat.yml的配置内容，否则报错：
错误1：
2023-05-15T08:45:51.149Z INFO instance/beat.go:328 Setup Beat: filebeat; Version: 7.17.6
2023-05-15T08:45:51.149Z INFO instance/beat.go:361 No outputs are defined. Please define one under the output section.
2023-05-15T08:45:51.149Z INFO instance/beat.go:461 filebeat stopped.
2023-05-15T08:45:51.149Z ERROR instance/beat.go:1014 Exiting: No outputs are defined. Please define one under the output section.
Exiting: No outputs are defined. Please define one under the output section.
错误2：
2023-05-15T09:03:09.492Z INFO [publisher] pipeline/module.go:113 Beat name: filebeat-62ql6
2023-05-15T09:03:09.493Z INFO instance/beat.go:461 filebeat stopped.
2023-05-15T09:03:09.493Z ERROR instance/beat.go:1014 Exiting: no modules or inputs enabled and configuration reloading disabled. What files do you want me to watch?
Exiting: no modules or inputs enabled and configuration reloading disabled. What files do you want me to watch?
错误3：
当master节点，内存还剩200MB的时候，部署daemonSet，他副本数会是0个， Events: <none>
正常运行后的情况：
模拟产生日志，并查看日志
删除node01节点上kube-system名称空间中的Pod，模拟产生日志
kubectl delete pod -n kube-system kube-proxy-6ks5b
kubectl logs -n logging filebeat-6qhwg
2.3 收集ingress-nginx名称空间
1、修改Ingress日志输出格式
kubectl edit configmaps -n ingress-nginx ingress-controller-leader-nginx
#加上data内容
log-format-upstream:'{"timestamp":"$time_iso8601","domain":"$server_name","hostname":"$hostname","remote_user":"$remote_user","clientip":"$remote_addr","proxy_protocol_addr":"$proxy_protocol_addr","@source":"$server_addr","host":"$http_host","request":"$request","args":"$args","upstreamaddr":"$upstream_addr","status":"$status","upstream_status":"$upstream_status","bytes":"$body_bytes_sent","responsetime":"$request_time","upstreamtime":"$upstream_response_time","proxy_upstream_name":"$proxy_upstream_name","x_forwarded":"$http_x_forwarded_for","upstream_response_length":"$upstream_response_length","referer":"$http_referer","user_agent":"$http_user_agent","request_length":"$request_length","request_method":"$request_method","scheme":"$scheme","k8s_ingress_name":"$ingress_name","k8s_service_name":"$service_name","k8s_service_port":"$service_port"}'
2、为filebeat增加如下内容（注意保留此前kube-system相关的配置）
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
data:
filebeat.yml: |-
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
- condition: # 1 匹配kube-system名称空间下所有日志
equals:
kubernetes.namespace: kube-system
config:
- type: container
stream: all
encoding: utf-8
paths: /var/log/containers/*-${data.kubernetes.container.id}.log
exclude_lines: ['info']
- condition: # 2 收集ingress-nginx命名空间下stdout日志
equals:
kubernetes.namespace: ingress-nginx
config:
- type: container
stream: stdout
encoding: utf-8
paths: /var/log/containers/*-${data.kubernetes.container.id}.log
json.keys_under_root: true #默认将json解析存储至messages，true则不存储至message
json.overwrite_keys: true #覆盖默认message字段，使用自定义json格式的key
- condition: # 3 收集ingress-nginx命名空间下stderr日志
equals:
kubernetes.namespace: ingress-nginx
config:
- type: container
stream: stderr
encoding: utf-8
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
output.console:
pretty: true
enable: true
3、访问ingress，模拟产生日志
先看ingress-nginx部署在那个节点上，然后logs -f那个节点上filebeat日志
curl kibana.oldxu.net:30080
kubectl logs -f -n logging filebeat-fqp84
2.4 收集kubelet本地应用程序日志
1、kubelet应用日志存储至每个节点 /var/log/messages中，所以直接追加如下一段静态方式收集即可；
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: logging
data:
filebeat.yml: |-
# ======= Filebeat inputs 静态方式收集
logging.level: warning
filebeat.inputs:
- type: log
enable: true
encoding: utf-8
paths: /var/log/messages
include_lines: ['kubelet'] # 4 获取与kubelet相关的日志
fields:
namespace: kubelet
fields_under_root: true
# ======= Filebeat autodiscover 动态方式收集
filebeat.autodiscover:
providers:
- type: kubernetes
templates:
- condition: # 1 匹配kube-system名称空间下所有日志
equals:
kubernetes.namespace: kube-system
config:
- type: container
stream: all
encoding: utf-8
paths: /var/log/containers/*-${data.kubernetes.container.id}.log
exclude_lines: ['info']
- condition: # 2 收集ingress-nginx命名空间下stdout日志
equals:
kubernetes.namespace: ingress-nginx
config:
- type: container
stream: stdout
encoding: utf-8
paths: /var/log/containers/*-${data.kubernetes.container.id}.log
json.keys_under_root: true #默认将json解析存储至messages，true则不存储至message
json.overwrite_keys: true #覆盖默认message字段，使用自定义json格式的key
- condition: # 3 收集ingress-nginx命名空间下stderr日志
equals:
kubernetes.namespace: ingress-nginx
config:
- type: container
stream: stderr
encoding: utf-8
paths:
- /var/log/containers/*-${data.kubernetes.container.id}.log
output.console:
pretty: true
enable: true
报错处理：
[root@master01 02-DaemonSet-agent-log]# kubectl logs -n logging filebeat-5kznt
Exiting: error loading config file: yaml: line 3: mapping values are not allowed in this context
2、检查filebeat，查看日志收集情况
/var/log/messages的kubelet相关内容：
filebeat的显示：
3.filebeat配置文件优化
3.1 优化filebeat输出段 与 修改配置输出至Kafka
优化filebeat输出段 :
当控制面板得到的信息符合预期时，我们需要将信息输出至Kafka，将output修改为如下内容即可；
