Skip to content

Commit c80e39d

Browse files
jianxiaoguojianxiaoguo
committed
chore(addons): modify kafka bind and fluentbit input
1 parent d3a318a commit c80e39d

8 files changed

Lines changed: 238 additions & 169 deletions

File tree

‎addons/fluentbit/2/chart/fluentbit/values.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,8 @@ daemonset:
184184
DB /data/containers.pos.db
185185
DB.locking true
186186
Offset_Key offset
187-
Tag kubernetes.*
187+
Tag kubernetes.<namespace_name>.<pod_name>.<container_name>.<container_id>
188+
Tag_Regex (?<pod_name>[a-z0-9](?:[-a-z0-9]*[a-z0-9])?(?:\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace_name>[^_]+)_(?<container_name>.+)-(?<container_id>[a-z0-9]{64})\.log$
188189
Read_from_Head false
189190
multiline.parser docker,cri
190191
## https://docs.fluentbit.io/manual/pipeline/outputs

‎addons/kafka/3.6/chart/kafka/templates/_helpers.tpl‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -575,6 +575,20 @@ Returns the zookeeper.connect setting value
575575
{{- end -}}
576576
{{- end -}}
577577

578+
{{/*
579+
Returns the internel listeners based on the number of controller-eligible nodes
580+
*/}}
581+
{{- define "kafka.kraft.internelListeners" -}}
582+
{{- $internelListeners := list -}}
583+
{{- $fullname := include "common.names.fullname" . -}}
584+
{{- $releaseNamespace := include "common.names.namespace" . -}}
585+
{{- range $i := until (int .Values.controller.replicaCount) -}}
586+
{{- $nodeAddress := printf "%s-controller-%d.%s-controller-headless.%s.svc.%s:%d" $fullname (int $i) $fullname $releaseNamespace $.Values.clusterDomain (int $.Values.listeners.interbroker.containerPort) -}}
587+
{{- $internelListeners = append $internelListeners (printf "%s" $nodeAddress ) -}}
588+
{{- end -}}
589+
{{- join "," $internelListeners -}}
590+
{{- end -}}
591+
578592
{{/*
579593
Returns the controller quorum voters based on the number of controller-eligible nodes
580594
*/}}

‎addons/kafka/3.6/plans/standard-16c32g3w/bind.yaml‎

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,12 @@
11
credential:
2-
{{- if (eq .Values.service.type "LoadBalancer") }}
3-
- name: EXTERNAL_HOST
4-
valueFrom:
5-
serviceRef:
6-
name: {{ template "common.names.fullname" . }}
7-
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
8-
{{- end }}
9-
10-
- name: CLIENT_HOST
11-
valueFrom:
12-
serviceRef:
13-
name: {{ template "common.names.fullname" . }}
14-
jsonpath: '{ .spec.clusterIP }'
2+
- name: INTERNAL_LISTENERS
3+
value: {{ include "kafka.kraft.internelListeners" . }}
154

16-
- name: CLIENT_PORT
17-
valueFrom:
18-
serviceRef:
19-
name: {{ template "common.names.fullname" . }}
20-
jsonpath: '{ .spec.ports[?(@.name=="tcp-client")].port }'
5+
- name: PROTOCOL_MAP
6+
value: {{ include "kafka.securityProtocolMap" . }}
217

22-
{{- if .Values.externalAccess.enabled }}
23-
- name: EXTERNAL_PORT
24-
valueFrom:
25-
serviceRef:
26-
name: {{ template "common.names.fullname" . }}
27-
jsonpath: '{ .spec.ports[?(@.name=="tcp-external")].port }'
28-
{{- end }}
29-
30-
31-
{{- if (include "kafka.createSaslSecret" .) }}
328
{{- if (include "kafka.saslUserPasswordsEnabled" .) }}
9+
{{- if (include "kafka.client.saslEnabled" .) }}
3310
- name: CLIENT_USERS
3411
value: {{ join "," .Values.sasl.client.users | quote }}
3512

@@ -45,4 +22,36 @@ credential:
4522
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
4623
jsonpath: '{ .data.system-user-password }'
4724
{{- end }}
25+
26+
{{- if regexFind "SASL" (upper .Values.listeners.interbroker.protocol) }}
27+
- name: INTER_BROKER_PASSPORTS
28+
valueFrom:
29+
secretKeyRef:
30+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
31+
jsonpath: '{ .data.inter-broker-password }'
32+
{{- end }}
33+
34+
{{- if regexFind "SASL" (upper .Values.listeners.controller.protocol) }}
35+
- name: CONTROLLER_PASSPORTS
36+
valueFrom:
37+
secretKeyRef:
38+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
39+
jsonpath: '{ .data.controller-password }'
4840
{{- end }}
41+
{{- end }}
42+
43+
{{- if .Values.externalAccess.enabled }}
44+
{{- $fullname := include "common.names.fullname" . }}
45+
{{- $replicaCount := .Values.controller.replicaCount | int }}
46+
{{- range $i := until $replicaCount }}
47+
{{- $targetPod := printf "%s-broker-%d" (printf "%s" $fullname) $i }}
48+
- name: {{ printf "EXTERNAL_%d" $i }}
49+
valueFrom:
50+
serviceRef:
51+
name: {{ printf "%s-broker-%d-external" (include "common.names.fullname" $) $i | trunc 63 | trimSuffix "-" }}
52+
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
53+
{{- end }}
54+
55+
- name: EXTERNAL_PORT
56+
value: {{ .Values.externalAccess.controller.service.ports.external }}
57+
{{- end }}

‎addons/kafka/3.6/plans/standard-1c2g3w/bind.yaml‎

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,12 @@
11
credential:
2-
{{- if (eq .Values.service.type "LoadBalancer") }}
3-
- name: EXTERNAL_HOST
4-
valueFrom:
5-
serviceRef:
6-
name: {{ template "common.names.fullname" . }}
7-
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
8-
{{- end }}
9-
10-
- name: CLIENT_HOST
11-
valueFrom:
12-
serviceRef:
13-
name: {{ template "common.names.fullname" . }}
14-
jsonpath: '{ .spec.clusterIP }'
2+
- name: INTERNAL_LISTENERS
3+
value: {{ include "kafka.kraft.internelListeners" . }}
154

16-
- name: CLIENT_PORT
17-
valueFrom:
18-
serviceRef:
19-
name: {{ template "common.names.fullname" . }}
20-
jsonpath: '{ .spec.ports[?(@.name=="tcp-client")].port }'
5+
- name: PROTOCOL_MAP
6+
value: {{ include "kafka.securityProtocolMap" . }}
217

22-
{{- if .Values.externalAccess.enabled }}
23-
- name: EXTERNAL_PORT
24-
valueFrom:
25-
serviceRef:
26-
name: {{ template "common.names.fullname" . }}
27-
jsonpath: '{ .spec.ports[?(@.name=="tcp-external")].port }'
28-
{{- end }}
29-
30-
31-
{{- if (include "kafka.createSaslSecret" .) }}
328
{{- if (include "kafka.saslUserPasswordsEnabled" .) }}
9+
{{- if (include "kafka.client.saslEnabled" .) }}
3310
- name: CLIENT_USERS
3411
value: {{ join "," .Values.sasl.client.users | quote }}
3512

@@ -45,4 +22,36 @@ credential:
4522
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
4623
jsonpath: '{ .data.system-user-password }'
4724
{{- end }}
25+
26+
{{- if regexFind "SASL" (upper .Values.listeners.interbroker.protocol) }}
27+
- name: INTER_BROKER_PASSPORTS
28+
valueFrom:
29+
secretKeyRef:
30+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
31+
jsonpath: '{ .data.inter-broker-password }'
32+
{{- end }}
33+
34+
{{- if regexFind "SASL" (upper .Values.listeners.controller.protocol) }}
35+
- name: CONTROLLER_PASSPORTS
36+
valueFrom:
37+
secretKeyRef:
38+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
39+
jsonpath: '{ .data.controller-password }'
4840
{{- end }}
41+
{{- end }}
42+
43+
{{- if .Values.externalAccess.enabled }}
44+
{{- $fullname := include "common.names.fullname" . }}
45+
{{- $replicaCount := .Values.controller.replicaCount | int }}
46+
{{- range $i := until $replicaCount }}
47+
{{- $targetPod := printf "%s-broker-%d" (printf "%s" $fullname) $i }}
48+
- name: {{ printf "EXTERNAL_%d" $i }}
49+
valueFrom:
50+
serviceRef:
51+
name: {{ printf "%s-broker-%d-external" (include "common.names.fullname" $) $i | trunc 63 | trimSuffix "-" }}
52+
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
53+
{{- end }}
54+
55+
- name: EXTERNAL_PORT
56+
value: {{ .Values.externalAccess.controller.service.ports.external }}
57+
{{- end }}

‎addons/kafka/3.6/plans/standard-24c64g3w/bind.yaml‎

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,12 @@
11
credential:
2-
{{- if (eq .Values.service.type "LoadBalancer") }}
3-
- name: EXTERNAL_HOST
4-
valueFrom:
5-
serviceRef:
6-
name: {{ template "common.names.fullname" . }}
7-
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
8-
{{- end }}
9-
10-
- name: CLIENT_HOST
11-
valueFrom:
12-
serviceRef:
13-
name: {{ template "common.names.fullname" . }}
14-
jsonpath: '{ .spec.clusterIP }'
2+
- name: INTERNAL_LISTENERS
3+
value: {{ include "kafka.kraft.internelListeners" . }}
154

16-
- name: CLIENT_PORT
17-
valueFrom:
18-
serviceRef:
19-
name: {{ template "common.names.fullname" . }}
20-
jsonpath: '{ .spec.ports[?(@.name=="tcp-client")].port }'
5+
- name: PROTOCOL_MAP
6+
value: {{ include "kafka.securityProtocolMap" . }}
217

22-
{{- if .Values.externalAccess.enabled }}
23-
- name: EXTERNAL_PORT
24-
valueFrom:
25-
serviceRef:
26-
name: {{ template "common.names.fullname" . }}
27-
jsonpath: '{ .spec.ports[?(@.name=="tcp-external")].port }'
28-
{{- end }}
29-
30-
31-
{{- if (include "kafka.createSaslSecret" .) }}
328
{{- if (include "kafka.saslUserPasswordsEnabled" .) }}
9+
{{- if (include "kafka.client.saslEnabled" .) }}
3310
- name: CLIENT_USERS
3411
value: {{ join "," .Values.sasl.client.users | quote }}
3512

@@ -45,4 +22,36 @@ credential:
4522
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
4623
jsonpath: '{ .data.system-user-password }'
4724
{{- end }}
25+
26+
{{- if regexFind "SASL" (upper .Values.listeners.interbroker.protocol) }}
27+
- name: INTER_BROKER_PASSPORTS
28+
valueFrom:
29+
secretKeyRef:
30+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
31+
jsonpath: '{ .data.inter-broker-password }'
32+
{{- end }}
33+
34+
{{- if regexFind "SASL" (upper .Values.listeners.controller.protocol) }}
35+
- name: CONTROLLER_PASSPORTS
36+
valueFrom:
37+
secretKeyRef:
38+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
39+
jsonpath: '{ .data.controller-password }'
4840
{{- end }}
41+
{{- end }}
42+
43+
{{- if .Values.externalAccess.enabled }}
44+
{{- $fullname := include "common.names.fullname" . }}
45+
{{- $replicaCount := .Values.controller.replicaCount | int }}
46+
{{- range $i := until $replicaCount }}
47+
{{- $targetPod := printf "%s-broker-%d" (printf "%s" $fullname) $i }}
48+
- name: {{ printf "EXTERNAL_%d" $i }}
49+
valueFrom:
50+
serviceRef:
51+
name: {{ printf "%s-broker-%d-external" (include "common.names.fullname" $) $i | trunc 63 | trimSuffix "-" }}
52+
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
53+
{{- end }}
54+
55+
- name: EXTERNAL_PORT
56+
value: {{ .Values.externalAccess.controller.service.ports.external }}
57+
{{- end }}

‎addons/kafka/3.6/plans/standard-2c4g3w/bind.yaml‎

Lines changed: 37 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,12 @@
11
credential:
2-
{{- if (eq .Values.service.type "LoadBalancer") }}
3-
- name: EXTERNAL_HOST
4-
valueFrom:
5-
serviceRef:
6-
name: {{ template "common.names.fullname" . }}
7-
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
8-
{{- end }}
9-
10-
- name: CLIENT_HOST
11-
valueFrom:
12-
serviceRef:
13-
name: {{ template "common.names.fullname" . }}
14-
jsonpath: '{ .spec.clusterIP }'
2+
- name: INTERNAL_LISTENERS
3+
value: {{ include "kafka.kraft.internelListeners" . }}
154

16-
- name: CLIENT_PORT
17-
valueFrom:
18-
serviceRef:
19-
name: {{ template "common.names.fullname" . }}
20-
jsonpath: '{ .spec.ports[?(@.name=="tcp-client")].port }'
5+
- name: PROTOCOL_MAP
6+
value: {{ include "kafka.securityProtocolMap" . }}
217

22-
{{- if .Values.externalAccess.enabled }}
23-
- name: EXTERNAL_PORT
24-
valueFrom:
25-
serviceRef:
26-
name: {{ template "common.names.fullname" . }}
27-
jsonpath: '{ .spec.ports[?(@.name=="tcp-external")].port }'
28-
{{- end }}
29-
30-
31-
{{- if (include "kafka.createSaslSecret" .) }}
328
{{- if (include "kafka.saslUserPasswordsEnabled" .) }}
9+
{{- if (include "kafka.client.saslEnabled" .) }}
3310
- name: CLIENT_USERS
3411
value: {{ join "," .Values.sasl.client.users | quote }}
3512

@@ -45,4 +22,36 @@ credential:
4522
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
4623
jsonpath: '{ .data.system-user-password }'
4724
{{- end }}
25+
26+
{{- if regexFind "SASL" (upper .Values.listeners.interbroker.protocol) }}
27+
- name: INTER_BROKER_PASSPORTS
28+
valueFrom:
29+
secretKeyRef:
30+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
31+
jsonpath: '{ .data.inter-broker-password }'
32+
{{- end }}
33+
34+
{{- if regexFind "SASL" (upper .Values.listeners.controller.protocol) }}
35+
- name: CONTROLLER_PASSPORTS
36+
valueFrom:
37+
secretKeyRef:
38+
name: {{ printf "%s-user-passwords" (include "common.names.fullname" .) }}
39+
jsonpath: '{ .data.controller-password }'
4840
{{- end }}
41+
{{- end }}
42+
43+
{{- if .Values.externalAccess.enabled }}
44+
{{- $fullname := include "common.names.fullname" . }}
45+
{{- $replicaCount := .Values.controller.replicaCount | int }}
46+
{{- range $i := until $replicaCount }}
47+
{{- $targetPod := printf "%s-broker-%d" (printf "%s" $fullname) $i }}
48+
- name: {{ printf "EXTERNAL_%d" $i }}
49+
valueFrom:
50+
serviceRef:
51+
name: {{ printf "%s-broker-%d-external" (include "common.names.fullname" $) $i | trunc 63 | trimSuffix "-" }}
52+
jsonpath: '{ .status.loadBalancer.ingress[*].ip }'
53+
{{- end }}
54+
55+
- name: EXTERNAL_PORT
56+
value: {{ .Values.externalAccess.controller.service.ports.external }}
57+
{{- end }}

0 commit comments

Comments
 (0)