使用kubeadm或者其他方式部署一套k8s集群。
在k8s集群创建一个namespace：halashow
2 ELK部署架构
3.1 准备资源配置清单
Deployment中存在一个es的业务容器，和一个init容器，init容器主要是配置vm.max_map_count=262144。
service暴露了9200端口，其他服务可通过service name加端口访问es。
3.1 准备资源配置清单
Deployment中存在一个es的业务容器，和一个init容器，init容器主要是配置vm.max_map_count=262144。
service暴露了9200端口，其他服务可通过service name加端口访问es。
apiVersion: v1
kind: Service
metadata:
namespace: halashow
name: elasticsearch
labels:
app: elasticsearch-logging
spec:
type: ClusterIP
ports:
- port: 9200
name: elasticsearch
selector:
app: elasticsearch-logging
---
apiVersion: apps/v1
kind: Deployment
metadata:
generation: 1
labels:
app: elasticsearch-logging
version: v1
name: elasticsearch
namespace: halashow
spec:
serviceName: elasticsearch-logging
minReadySeconds: 10
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: elasticsearch-logging
version: v1
strategy:
type: Recreate
template:
metadata:
creationTimestamp: null
labels:
app: elasticsearch-logging
version: v1
spec:
affinity:
nodeAffinity: {}
containers:
- env:
- name: discovery.type
value: single-node
- name: ES_JAVA_OPTS
value: -Xms512m -Xmx512m
- name: MINIMUM_MASTER_NODES
value: "1"
image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0-amd64
imagePullPolicy: IfNotPresent
name: elasticsearch-logging
ports:
- containerPort: 9200
name: db
protocol: TCP
- containerPort: 9300
name: transport
protocol: TCP
resources:
limits:
cpu: "1"
memory: 1Gi
requests:
cpu: "1"
memory: 1Gi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /data
name: es-persistent-storage
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: user-1-registrysecret
initContainers:
- command:
- /sbin/sysctl
- -w
- vm.max_map_count=262144
image: alpine:3.6
imagePullPolicy: IfNotPresent
name: elasticsearch-logging-init
resources: {}
securityContext:
privileged: true
procMount: Default
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- hostPath:
path: /data/elk/elasticsearch-logging
type: DirectoryOrCreate
name: es-persistent-storage
nodeSelector:
alibabacloud.com/is-edge-worker: 'false'
beta.kubernetes.io/arch: amd64
beta.kubernetes.io/os: linux
tolerations:
- effect: NoSchedule
key: node-role.alibabacloud.com/addon
operator: Exists
4 部署logstash
创建configMap定义logstash相关配置项，主要包括一下几项。
input：定义输入到logstash的源。
filter：定义过滤条件。
output：可以定义输出到es，redis，kafka等等。
---
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: halashow
data:
logstash.conf: |-
input {
beats {
port => 5044
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch {
hosts => ["elasticsearch:9200"]
}
}
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
namespace: halashow
labels:
name: logstash
spec:
replicas: 1
selector:
matchLabels:
name: logstash
template:
metadata:
labels:
app: logstash
name: logstash
spec:
containers:
- name: logstash
image: docker.elastic.co/logstash/logstash:7.12.0
ports:
- containerPort: 5044
protocol: TCP
- containerPort: 9600
protocol: TCP
volumeMounts:
- name: logstash-config
#mountPath: /usr/share/logstash/logstash-simple.conf
#mountPath: /usr/share/logstash/config/logstash-sample.conf
mountPath: /usr/share/logstash/pipeline/logstash.conf
subPath: logstash.conf
#ports:
# - containerPort: 80
# protocol: TCP
volumes:
- name: logstash-config
configMap:
#defaultMode: 0644
name: logstash-config
---
apiVersion: v1
kind: Service
metadata:
namespace: halashow
name: logstash
labels:
app: logstash
spec:
type: ClusterIP
ports:
- port: 5044
name: logstash
selector:
app: logstash5 部署filebeat
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config-to-logstash
namespace: halashow
data:
filebeat.yml: |-
filebeat.inputs:
- type: log
paths:
- /logm/*.log
output.logstash:
hosts: ['logstash:5044']
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: filebeat
namespace: halashow
labels:
name: filebeat
spec:
replicas: 1
selector:
matchLabels:
name: filebeat
template:
metadata:
labels:
app: filebeat
name: filebeat
spec:
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:7.12.0
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
volumeMounts:
- mountPath: /logm
name: logm
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
volumes:
- name: logm
emptyDir: {}
- name: config
configMap:
defaultMode: 0640
name: filebeat-config-to-logstash
6 部署kibana
6.1 准备资源配置清单
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: halashow
labels:
name: kibana
spec:
serviceName: halashow
replicas: 1
selector:
matchLabels:
name: kibana
template:
metadata:
labels:
app: kibana
name: kibana
spec:
restartPolicy: Always
containers:
- name: kibana
image: kibana:7.12.0
imagePullPolicy: Always
ports:
- containerPort: 5601
resources:
requests:
memory: 1024Mi
cpu: 50m
limits:
memory: 1024Mi
cpu: 1000m
volumeMounts:
- name: kibana-config
mountPath: /usr/share/kibana/config/kibana.yml
subPath: kibana.yml
volumes:
- name: kibana-config
configMap:
name: kibana-cm
items:
- key: "kibana.yml"
path: "kibana.yml"
---
apiVersion: v1
kind: Service
metadata:
labels:
app: kibana
name: kibana
namespace: halashow
spec:
type: NodePort
ports:
- name: kibana
port: 5601
nodePort: 30102
protocol: TCP
targetPort: 5601
selector:
app: kibana
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN" #kibana汉化
location / #必须是/否则代码不上
passpoxry http:ip:port
