Skip to content

Commit bc088c1

Browse files
EamonZhangEamonZhang
committed
chore(fluentbit): Redefining Configuration Files
2 parents 24b036f + 29fb7ab commit bc088c1

4 files changed

Lines changed: 69 additions & 93 deletions

File tree

‎addons/fluentbit/2/chart/fluentbit/templates/configmap.yaml‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,24 @@ metadata:
1414
{{- end }}
1515
data:
1616
parsers.conf: |
17+
{{- (tpl .Values.daemonset.config.parsers_base $) | nindent 4 }}
1718
{{- (tpl .Values.daemonset.config.parsers $) | nindent 4 }}
1819
fluentbit.conf: |
19-
{{- (tpl .Values.daemonset.config.service $) | nindent 4 }}
20-
{{- (tpl .Values.daemonset.config.inputs $) | nindent 4 }}
21-
{{- (tpl .Values.daemonset.config.outputs $) | nindent 4 }}
20+
{{- $service := (tpl .Values.daemonset.config.service $) }}
21+
{{- if not (contains "[INPUT]" $service) }}
22+
{{- $service | nindent 4 }}
23+
{{- end }}
24+
25+
{{- (tpl .Values.daemonset.config.inputs $) | nindent 4 }}
26+
27+
{{- (tpl .Values.daemonset.config.filters_base $) | nindent 4 }}
28+
29+
{{- $filters := (tpl .Values.daemonset.config.filters $) }}
30+
{{- if not (contains "[INPUT]" $filters) }}
31+
{{- $filters | nindent 4 }}
32+
{{- end }}
33+
34+
{{- $outputs := (tpl .Values.daemonset.config.outputs $) }}
35+
{{- if not (contains "[INPUT]" $outputs) }}
36+
{{- (tpl .Values.daemonset.config.outputs $) | nindent 4 }}
37+
{{- end }}

‎addons/fluentbit/2/chart/fluentbit/values.yaml‎

Lines changed: 44 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -165,107 +165,67 @@ daemonset:
165165
flush: 1
166166
## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file
167167
config:
168-
multilines : python_multiline,java_multiline,go_multiline,gateway_multiline
168+
169169
service: |
170170
[SERVICE]
171-
Flush {{ .Values.daemonset.flush }}
172-
Daemon Off
173-
Log_Level info
174-
HTTP_Server On
175-
HTTP_Listen 127.0.0.1
176-
HTTP_PORT 2020
177-
Hot_Reload On
178-
Parsers_File /opt/drycc/fluent-bit/etc/fluent-bit/parsers.conf
171+
Flush {{ .Values.daemonset.flush }}
172+
Daemon Off
173+
Log_Level info
174+
HTTP_Server On
175+
HTTP_Listen 127.0.0.1
176+
HTTP_PORT 2020
177+
Hot_Reload On
178+
Parsers_File /opt/drycc/fluent-bit/etc/fluent-bit/parsers.conf
179179
180180
## https://docs.fluentbit.io/manual/pipeline/inputs
181181
inputs: |
182182
[INPUT]
183-
Name tail
184-
Path /var/log/containers/*_{{ .Release.Namespace }}_*.log
185-
DB /data/containers.pos.db
186-
DB.locking true
187-
Offset_Key offset
188-
Tag kubernetes.<namespace_name>.<pod_name>.<container_name>.<container_id>
189-
Tag_Regex (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id>[a-z0-9]{64})\.log$
190-
Read_from_Head false
191-
multiline.parser docker,cri
183+
Name tail
184+
Path /var/log/containers/*_{{ .Release.Namespace }}_*.log
185+
DB /data/containers.pos.db
186+
DB.locking true
187+
Offset_Key offset
188+
Tag kubernetes.<namespace_name>.<pod_name>.<container_name>.<container_id>
189+
Tag_Regex (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id>[a-z0-9]{64})\.log$
190+
Read_from_Head false
191+
multiline.parser docker,cri
192192
193+
## https://docs.fluentbit.io/manual/pipeline/filters
194+
filters_base: |
193195
[FILTER]
194-
Name Lua
195-
Match kubernetes.*
196-
call append_tag
197-
code function append_tag(tag, timestamp, record) new_record = record local namespace, pod_name, container_name, container_id = string.match(tag, "kubernetes%.([^%.]+)%.([^%.]+)%.([^%.]+)%.(.+)") if namespace and pod_name and container_name and container_id then new_record["pod_name"] = pod_name new_record["namespace"] = namespace new_record["container_name"] = container_name new_record["container_id"] = container_id end return 1, timestamp, new_record end
198-
199-
[FILTER]
200-
name multiline
201-
match kubernetes.*
202-
multiline.parser {{ .Values.daemonset.config.multilines }}
203-
multiline.key_content log
204-
buffer on
196+
Name Lua
197+
Match kubernetes.*
198+
call append_tag
199+
code function append_tag(tag, timestamp, record) new_record = record local namespace, pod_name, container_name, container_id = string.match(tag, "kubernetes%.([^%.]+)%.([^%.]+)%.([^%.]+)%.(.+)") if namespace and pod_name and container_name and container_id then new_record["pod_name"] = pod_name new_record["namespace"] = namespace new_record["container_name"] = container_name new_record["container_id"] = container_id end return 1, timestamp, new_record end
200+
201+
filters: |
205202
206203
## https://docs.fluentbit.io/manual/pipeline/outputs
207204
outputs: |
208205
[OUTPUT]
209-
Name stdout
210-
Match kubernetes.*
206+
Name stdout
207+
Match kubernetes.*
208+
211209
## https://docs.fluentbit.io/manual/pipeline/parsers
212-
parsers: |
210+
parsers_base: |
213211
[PARSER]
214-
Name docker
215-
Format json
216-
Time_Key time
217-
Time_Format %Y-%m-%dT%H:%M:%S.%L
218-
Time_Keep On
212+
Name docker
213+
Format json
214+
Time_Key time
215+
Time_Format %Y-%m-%dT%H:%M:%S.%L
216+
Time_Keep On
219217
220218
[PARSER]
221219
# http://rubular.com/r/tjUt3Awgg4
222-
Name cri
223-
Format regex
224-
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
225-
Time_Key time
226-
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
227-
Time_Keep On
228-
229-
[MULTILINE_PARSER]
230-
Name python_multiline
231-
Type regex
232-
Flush_Timeout 1000
233-
key_content log
234-
Rule "start_state" "/^\[.*ERROR.*-- Internal Server Error: .*$/" "cont"
235-
Rule "cont" "/^Traceback \(most recent call last\):$/" "cont"
236-
Rule "cont" "/^\s+File \"/" "cont"
237-
Rule "cont" "/^\s+raise /" "cont"
238-
Rule "cont" "/^\s?\w+Error: /" "cont"
239-
Rule "cont" "/^\s+/" "cont"
240-
241-
[MULTILINE_PARSER]
242-
Name java_multiline
243-
Type regex
244-
Flush_Timeout 1000
245-
key_content log
246-
Rule "start_state" "/^\d{4}-\d{2}-\d{2}/" "cont"
247-
Rule "cont" "/^\s+at /" "cont"
248-
Rule "cont" "/^[a-z]/" "cont"
249-
Rule "cont" "/^Caused by:/" "cont"
250-
Rule "cont" "/^\s+/" "cont"
251-
Rule "cont" "/^(?!\d{4}-\d{2}-\d{2})/" "cont"
252-
253-
[MULTILINE_PARSER]
254-
Name go_multiline
255-
Type regex
256-
Flush_Timeout 1000
257-
key_content log
258-
Rule "start_state" "/^panic: /" "cont"
259-
Rule "cont" "/^\s+/" "cont"
260-
Rule "cont" "/^goroutine /" "cont"
261-
262-
[MULTILINE_PARSER]
263-
Name gateway_multiline
264-
Type regex
265-
Flush_Timeout 1000
266-
key_content log
267-
Rule "start_state" "/^\[\d{4}-\d{2}-\d{2}/" "cont"
268-
Rule "cont" "/^(?!\[\d{4}-\d{2}-\d{2})/" "cont"
220+
Name cri
221+
Format regex
222+
Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<message>.*)$
223+
Time_Key time
224+
Time_Format %Y-%m-%dT%H:%M:%S.%L%z
225+
Time_Keep On
226+
227+
parsers: |
228+
269229
270230
## @param daemonset.extraEnvVars Array with extra environment variables to add to daemonset nodes
271231
## e.g:

‎addons/fluentbit/2/meta.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@ allow_parameters:
2121
- name: "daemonset.config.outputs"
2222
required: true
2323
description: "destinations for your data: databases, cloud services and more"
24-
- name: "daemonset.config.multilines"
25-
equired: false
26-
description: "destinations multilines"
2724
- name: "daemonset.config.parsers"
2825
equired: false
2926
description: "destinations parsers"
27+
- name: "daemonset.config.filters"
28+
equired: false
29+
description: "fluentbit filters config"
3030
archive: false

‎addons/fluentbit/2/plans/standard-200m/meta.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
name: "standard-5"
1+
name: "standard-200m"
22
id: 1b3968b6-ea84-4f34-b1d9-b51e65d01b1c
3-
description: "Fluentbit standard-5 plan: vCPUs 100m, RAM 128M"
4-
displayName: "standard-5"
3+
description: "Fluentbit standard-200m plan: vCPUs 200m, RAM 128M"
4+
displayName: "standard-200m"
55
bindable: true
66
maximum_polling_duration: 1800

0 commit comments

Comments
 (0)