fluentd配置原创
# 采集k8s容器标准输出日志
<source>
@type tail
path /var/log/containers/kube-*.log,/var/log/containers/myapp-*.log # 逗号分隔
exclude_path ["/var/log/containers/kubectl-*.log"] # 排除的日志
pos_file /var/log/pos/containers.log.pos # 保存已读日志文件的位置
tag raw.*
read_from_head true
<parse>
@type multi_format
# 匹配docker容器日志格式json
<pattern>
format json
time_key time
time_format %Y-%m-%dT%H:%M:%S.%NZ
</pattern>
#这部分用来正则匹配CRI容器日志格式text
<pattern>
format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
time_format %Y-%m-%dT%H:%M:%S.%N%:z
</pattern>
</parse>
</source>
<filter raw.**>
@type kubernetes_metadata
@id kubernetes_metadata_container_out
skip_container_metadata true
skip_master_url true
cache_size 3000
cache_ttl 1800
</filter>
<match raw.**>
@type record_modifier
@id label.container.out
tag k8s-containers-${record.dig("k8s_container_name")}
# tag ${record.dig('log-collect') ? k8s-containers-${record.dig("k8s_container_name")} : 'dropped.raw'}
<record>
k8s_container_id ${record.dig("docker", "container_id")}
k8s_cloud_cluster "#{ENV['CLOUD_CLUSTER'] || 'default'}"
k8s_node ${record.dig('kubernetes', 'host')}
k8s_container_name ${record.dig('kubernetes', 'container_name')}
k8s_pod_name ${record.dig('kubernetes', 'pod_name')}
k8s_namespace_name ${record.dig('kubernetes', 'namespace_name')}
# log-collect ${record.dig("kubernetes", "labels", "log-collect") or false}
formated_time "${Time.at(time).to_datetime.iso8601(9)}"
fluentd_worker "#{worker_id}"
</record>
remove_keys docker,kubernetes # 删除原生metadata字段
</match>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
# 本地日志文件-主机审计日志
<source>
@type tail
read_from_head false
path /var/log/operation/*.log # 宿主机操作日志
follow_inodes true # 如果没有这个参数,文件轮转会导致日志重复
path_key file_name # 指定事件中path的key名称
pos_file /var/log/pos/operation.log.pos # 保存已读日志文件的位置
tag host-operation
limit_recently_modified 7d # 只监控指定修改时间范围内的文件
multiline_flush_interval 1s # 多行处理模式下的缓存输出间隔
<parse>
@type json
time_key time # 从事件的什么字段中获取时间,如果该字段不存在,则取当前时间
time_type string # 可选值:float:UNIX时间.纳秒、unixtime: UNIX时间(秒)、string:根据后面几个参数决定具体格式
time_format %Y-%m-%d %H:%M:%S # 时间格式
timezone +08:00
</parse>
</source>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# journal日志
<source>
@type systemd
path /run/log/journal
matches [{ "_SYSTEMD_UNIT": ["kubelet.service", "etcd.service"] }] # 指定journald日志,屏蔽此行则采集所有journal日志
<storage>
@type local
persistent true
path /var/log/pos/journal.pos
</storage>
read_from_head true
tag journald
</source>
1
2
3
4
5
6
7
8
9
10
11
12
2
3
4
5
6
7
8
9
10
11
12
# 输出
# <match dropped.**>
# @type null # 需要忽略的日志
# </match>
<filter k8s-*>
@type record_transformer
<record>
nodename "#{ENV['K8S_NODE_NAME']}" # 为每条记录附加节点字段
dc "#{ENV['CLUSTER_NAME']}" # 为每条记录附加集群名
</record>
</filter>
<match k8s-*> # 存储到es地址
@type forward
<server>
host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
</server>
<buffer tag,time>
@type file
path /var/log/td-agent/buffer/docker-k8s
timekey 3600
timekey_wait 0s
timekey_use_utc false
flush_mode interval
flush_interval 10s
chunk_limit_size 20M
</buffer>
</match>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
上次更新: 2022/10/09, 17:42:01