Skip to content

Commit 2715230

Browse files
jianxiaoguojianxiaoguo
committed
feat(kafka): add kafka with zookeeper
1 parent 01a26c9 commit 2715230

39 files changed

Lines changed: 5676 additions & 1 deletion

‎addons/index.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,7 @@ entries:
1212
- name: redis
1313
description: "Redis is an advanced key-value cache and store."
1414
version: 1.1.0
15+
kafka:
16+
- name: kafka
17+
description: "Apache Kafka is an open-source distributed event streaming platform used by thousands of companies for high-performance data pipelines, streaming analytics, data integration, and mission-critical applications."
18+
version: 1.1.0

‎addons/kafka-1.1.0/chart/kafka/.helmignore‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*~
18+
# Various IDEs
19+
.project
20+
.idea/
21+
*.tmproj

‎addons/kafka-1.1.0/chart/kafka/Chart.lock‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
dependencies:
2+
- name: zookeeper
3+
repository: https://charts.bitnami.com/bitnami
4+
version: 9.2.5
5+
- name: common
6+
repository: https://charts.bitnami.com/bitnami
7+
version: 1.16.0
8+
digest: sha256:5e2e8c54b17911cc226ab44c917d8a0d2d3a177cd972ea51aaf9ab0fa3ff008b
9+
generated: "2022-06-11T05:32:07.200687375Z"

‎addons/kafka-1.1.0/chart/kafka/Chart.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
annotations:
2+
category: Infrastructure
3+
apiVersion: v2
4+
appVersion: 3.2.0
5+
dependencies:
6+
- condition: zookeeper.enabled
7+
name: zookeeper
8+
repository: https://charts.bitnami.com/bitnami
9+
version: 9.x.x
10+
- name: common
11+
repository: https://charts.bitnami.com/bitnami
12+
tags:
13+
- bitnami-common
14+
version: 1.x.x
15+
description: Apache Kafka is a distributed streaming platform designed to build real-time pipelines and can be used as a message broker or as a replacement for a log aggregation solution for big data applications.
16+
engine: gotpl
17+
home: https://github.com/bitnami/charts/tree/master/bitnami/kafka
18+
icon: https://bitnami.com/assets/stacks/kafka/img/kafka-stack-220x234.png
19+
keywords:
20+
- kafka
21+
- zookeeper
22+
- streaming
23+
- producer
24+
- consumer
25+
maintainers:
26+
- name: Bitnami
27+
url: https://github.com/bitnami/charts
28+
name: kafka
29+
sources:
30+
- https://github.com/bitnami/bitnami-docker-kafka
31+
- https://kafka.apache.org/
32+
version: 17.2.6

‎addons/kafka-1.1.0/chart/kafka/README.md‎

Lines changed: 992 additions & 0 deletions
Large diffs are not rendered by default.

‎addons/kafka-1.1.0/chart/kafka/templates/NOTES.txt‎

Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
CHART NAME: {{ .Chart.Name }}
2+
CHART VERSION: {{ .Chart.Version }}
3+
APP VERSION: {{ .Chart.AppVersion }}
4+
5+
{{- if .Values.diagnosticMode.enabled }}
6+
The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with:
7+
8+
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }}
9+
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }}
10+
11+
Get the list of pods by executing:
12+
13+
kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }}
14+
15+
Access the pod you want to debug by executing
16+
17+
kubectl exec --namespace {{ .Release.Namespace }} -ti <NAME OF THE POD> -- bash
18+
19+
In order to replicate the container startup scripts execute this command:
20+
21+
/opt/drycc/scripts/kafka/entrypoint.sh /opt/drycc/scripts/kafka/run.sh
22+
23+
{{- else }}
24+
25+
{{- $replicaCount := int .Values.replicaCount -}}
26+
{{- $releaseNamespace := .Release.Namespace -}}
27+
{{- $clusterDomain := .Values.clusterDomain -}}
28+
{{- $fullname := include "common.names.fullname" . -}}
29+
{{- $clientProtocol := include "kafka.listenerType" (dict "protocol" .Values.auth.clientProtocol) -}}
30+
{{- $externalClientProtocol := include "kafka.listenerType" (dict "protocol" (include "kafka.externalClientProtocol" . )) -}}
31+
{{- $saslMechanisms := .Values.auth.sasl.mechanisms -}}
32+
{{- $tlsEndpointIdentificationAlgorithm := default "" .Values.auth.tls.endpointIdentificationAlgorithm -}}
33+
{{- $tlsPasswordSecret := printf "$(kubectl get secret %s --namespace %s -o jsonpath='{.data.password}' | base64 -d | cut -d , -f 1)" .Values.auth.tls.existingSecret $releaseNamespace -}}
34+
{{- $tlsPassword := ternary .Values.auth.tls.password $tlsPasswordSecret (eq .Values.auth.tls.existingSecret "") -}}
35+
{{- $servicePort := int .Values.service.ports.client -}}
36+
37+
{{- if and (or (eq .Values.service.type "LoadBalancer") .Values.externalAccess.enabled) (eq $externalClientProtocol "PLAINTEXT") }}
38+
---------------------------------------------------------------------------------------------
39+
WARNING
40+
41+
By specifying "serviceType=LoadBalancer" and not configuring the authentication
42+
you have most likely exposed the Kafka service externally without any
43+
authentication mechanism.
44+
45+
For security reasons, we strongly suggest that you switch to "ClusterIP" or
46+
"NodePort". As alternative, you can also configure the Kafka authentication.
47+
48+
---------------------------------------------------------------------------------------------
49+
{{- end }}
50+
51+
** Please be patient while the chart is being deployed **
52+
53+
Kafka can be accessed by consumers via port {{ $servicePort }} on the following DNS name from within your cluster:
54+
55+
{{ $fullname }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}
56+
57+
Each Kafka broker can be accessed by producers via port {{ $servicePort }} on the following DNS name(s) from within your cluster:
58+
59+
{{- $brokerList := list }}
60+
{{- range $e, $i := until $replicaCount }}
61+
{{- $brokerList = append $brokerList (printf "%s-%d.%s-headless.%s.svc.%s:%d" $fullname $i $fullname $releaseNamespace $clusterDomain $servicePort) }}
62+
{{- end }}
63+
{{ join "\n" $brokerList | nindent 4 }}
64+
{{- if (include "kafka.client.saslAuthentication" .) }}
65+
66+
You need to configure your Kafka client to access using SASL authentication. To do so, you need to create the 'kafka_jaas.conf' and 'client.properties' configuration files with the content below:
67+
68+
- kafka_jaas.conf:
69+
70+
KafkaClient {
71+
{{- if $saslMechanisms | regexFind "scram" }}
72+
org.apache.kafka.common.security.scram.ScramLoginModule required
73+
{{- else }}
74+
org.apache.kafka.common.security.plain.PlainLoginModule required
75+
{{- end }}
76+
username="{{ index .Values.auth.sasl.jaas.clientUsers 0 }}"
77+
password="$(kubectl get secret {{ $fullname }}-jaas --namespace {{ $releaseNamespace }} -o jsonpath='{.data.client-passwords}' | base64 -d | cut -d , -f 1)";
78+
};
79+
80+
- client.properties:
81+
82+
security.protocol={{ $clientProtocol }}
83+
{{- if $saslMechanisms | regexFind "scram-sha-256" }}
84+
sasl.mechanism=SCRAM-SHA-256
85+
{{- else if $saslMechanisms | regexFind "scram-sha-512" }}
86+
sasl.mechanism=SCRAM-SHA-512
87+
{{- else }}
88+
sasl.mechanism=PLAIN
89+
{{- end }}
90+
{{- if eq $clientProtocol "SASL_SSL" }}
91+
ssl.truststore.type={{ upper .Values.auth.tls.type }}
92+
{{- if eq .Values.auth.tls.type "jks" }}
93+
ssl.truststore.location=/tmp/kafka.truststore.jks
94+
{{- if not (empty $tlsPassword) }}
95+
ssl.truststore.password={{ $tlsPassword }}
96+
{{- end }}
97+
{{- else if eq .Values.auth.tls.type "pem" }}
98+
ssl.truststore.certificates=-----BEGIN CERTIFICATE----- \
99+
... \
100+
-----END CERTIFICATE-----
101+
{{- end }}
102+
{{- if eq $tlsEndpointIdentificationAlgorithm "" }}
103+
ssl.endpoint.identification.algorithm=
104+
{{- end }}
105+
{{- end }}
106+
107+
{{- else if (include "kafka.client.tlsEncryption" .) }}
108+
109+
You need to configure your Kafka client to access using TLS authentication. To do so, you need to create the 'client.properties' configuration file with the content below:
110+
111+
security.protocol={{ $clientProtocol }}
112+
ssl.truststore.type={{ upper .Values.auth.tls.type }}
113+
{{- if eq .Values.auth.tls.type "jks" }}
114+
ssl.truststore.location=/tmp/kafka.truststore.{{ .Values.auth.tls.type }}
115+
{{- if not (empty $tlsPassword) }}
116+
ssl.truststore.password={{ $tlsPassword }}
117+
{{- end }}
118+
{{- else if eq .Values.auth.tls.type "pem" }}
119+
ssl.truststore.certificates=-----BEGIN CERTIFICATE----- \
120+
... \
121+
-----END CERTIFICATE-----
122+
{{- end }}
123+
{{- if eq .Values.auth.clientProtocol "mtls" }}
124+
ssl.keystore.type={{ upper .Values.auth.tls.type }}
125+
{{- if eq .Values.auth.tls.type "jks" }}
126+
ssl.keystore.location=/tmp/client.keystore.jks
127+
{{- if not (empty $tlsPassword) }}
128+
ssl.keystore.password={{ $tlsPassword }}
129+
{{- end }}
130+
{{- else if eq .Values.auth.tls.type "pem" }}
131+
ssl.keystore.certificate.chain=-----BEGIN CERTIFICATE----- \
132+
... \
133+
-----END CERTIFICATE-----
134+
ssl.keystore.key=-----BEGIN ENCRYPTED PRIVATE KEY----- \
135+
... \
136+
-----END ENCRYPTED PRIVATE KEY-----
137+
{{- end }}
138+
{{- end }}
139+
{{- if eq $tlsEndpointIdentificationAlgorithm "" }}
140+
ssl.endpoint.identification.algorithm=
141+
{{- end }}
142+
143+
{{- end }}
144+
145+
To create a pod that you can use as a Kafka client run the following commands:
146+
147+
kubectl run {{ $fullname }}-client --restart='Never' --image {{ template "kafka.image" . }} --namespace {{ $releaseNamespace }} --command -- sleep infinity
148+
{{- if or (include "kafka.client.saslAuthentication" .) (include "kafka.client.tlsEncryption" .) }}
149+
kubectl cp --namespace {{ $releaseNamespace }} /path/to/client.properties {{ $fullname }}-client:/tmp/client.properties
150+
{{- end }}
151+
{{- if (include "kafka.client.saslAuthentication" .) }}
152+
kubectl cp --namespace {{ $releaseNamespace }} /path/to/kafka_jaas.conf {{ $fullname }}-client:/tmp/kafka_jaas.conf
153+
{{- end }}
154+
{{- if and (include "kafka.client.tlsEncryption" .) (eq .Values.auth.tls.type "jks") }}
155+
kubectl cp --namespace {{ $releaseNamespace }} ./kafka.truststore.jks {{ $fullname }}-client:/tmp/kafka.truststore.jks
156+
{{- if eq .Values.auth.clientProtocol "mtls" }}
157+
kubectl cp --namespace {{ $releaseNamespace }} ./client.keystore.jks {{ $fullname }}-client:/tmp/client.keystore.jks
158+
{{- end }}
159+
{{- end }}
160+
kubectl exec --tty -i {{ $fullname }}-client --namespace {{ $releaseNamespace }} -- bash
161+
{{- if (include "kafka.client.saslAuthentication" .) }}
162+
export KAFKA_OPTS="-Djava.security.auth.login.config=/tmp/kafka_jaas.conf"
163+
{{- end }}
164+
165+
PRODUCER:
166+
kafka-console-producer.sh \
167+
{{ if or (include "kafka.client.saslAuthentication" .) (include "kafka.client.tlsEncryption" .) }}--producer.config /tmp/client.properties \{{ end }}
168+
--broker-list {{ join "," $brokerList }} \
169+
--topic test
170+
171+
CONSUMER:
172+
kafka-console-consumer.sh \
173+
{{ if or (include "kafka.client.saslAuthentication" .) (include "kafka.client.tlsEncryption" .) }}--consumer.config /tmp/client.properties \{{ end }}
174+
--bootstrap-server {{ $fullname }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ .Values.service.ports.client }} \
175+
--topic test \
176+
--from-beginning
177+
178+
{{- if .Values.externalAccess.enabled }}
179+
180+
To connect to your Kafka server from outside the cluster, follow the instructions below:
181+
182+
{{- if eq "NodePort" .Values.externalAccess.service.type }}
183+
{{- if .Values.externalAccess.service.domain }}
184+
185+
Kafka brokers domain: Use your provided hostname to reach Kafka brokers, {{ .Values.externalAccess.service.domain }}
186+
187+
{{- else }}
188+
189+
Kafka brokers domain: You can get the external node IP from the Kafka configuration file with the following commands (Check the EXTERNAL listener)
190+
191+
1. Obtain the pod name:
192+
193+
kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka"
194+
195+
2. Obtain pod configuration:
196+
197+
kubectl exec -it KAFKA_POD -- cat /opt/drycc/kafka/config/server.properties | grep advertised.listeners
198+
199+
{{- end }}
200+
201+
Kafka brokers port: You will have a different node port for each Kafka broker. You can get the list of configured node ports using the command below:
202+
203+
echo "$(kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -o jsonpath='{.items[*].spec.ports[0].nodePort}' | tr ' ' '\n')"
204+
205+
{{- else if contains "LoadBalancer" .Values.externalAccess.service.type }}
206+
207+
NOTE: It may take a few minutes for the LoadBalancer IPs to be available.
208+
Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -w'
209+
210+
Kafka Brokers domain: You will have a different external IP for each Kafka broker. You can get the list of external IPs using the command below:
211+
212+
echo "$(kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -o jsonpath='{.items[*].status.loadBalancer.ingress[0].ip}' | tr ' ' '\n')"
213+
214+
Kafka Brokers port: {{ .Values.externalAccess.service.ports.external }}
215+
216+
{{- end }}
217+
218+
{{- if not (eq $clientProtocol $externalClientProtocol) }}
219+
{{- if (include "kafka.client.saslAuthentication" .) }}
220+
221+
You need to configure your Kafka client to access using SASL authentication. To do so, you need to create the 'kafka_jaas.conf' and 'client.properties' configuration files with the content below:
222+
223+
- kafka_jaas.conf:
224+
225+
KafkaClient {
226+
{{- if $saslMechanisms | regexFind "scram" }}
227+
org.apache.kafka.common.security.scram.ScramLoginModule required
228+
{{- else }}
229+
org.apache.kafka.common.security.plain.PlainLoginModule required
230+
{{- end }}
231+
username="{{ index .Values.auth.sasl.jaas.clientUsers 0 }}"
232+
password="$(kubectl get secret {{ $fullname }}-jaas --namespace {{ $releaseNamespace }} -o jsonpath='{.data.client-passwords}' | base64 -d | cut -d , -f 1)";
233+
};
234+
235+
- client.properties:
236+
237+
security.protocol={{ $externalClientProtocol }}
238+
{{- if $saslMechanisms | regexFind "scram-sha-256" }}
239+
sasl.mechanism=SCRAM-SHA-256
240+
{{- else if $saslMechanisms | regexFind "scram-sha-512" }}
241+
sasl.mechanism=SCRAM-SHA-512
242+
{{- else }}
243+
sasl.mechanism=PLAIN
244+
{{- end }}
245+
{{- if eq $externalClientProtocol "SASL_SSL" }}
246+
ssl.truststore.type={{ upper .Values.auth.tls.type }}
247+
{{- if eq .Values.auth.tls.type "jks" }}
248+
ssl.truststore.location=/tmp/kafka.truststore.jks
249+
{{- if not (empty $tlsPassword) }}
250+
ssl.truststore.password={{ $tlsPassword }}
251+
{{- end }}
252+
{{- else if eq .Values.auth.tls.type "pem" }}
253+
ssl.truststore.certificates=-----BEGIN CERTIFICATE----- \
254+
... \
255+
-----END CERTIFICATE-----
256+
{{- end }}
257+
{{- if eq $tlsEndpointIdentificationAlgorithm "" }}
258+
ssl.endpoint.identification.algorithm=
259+
{{- end }}
260+
{{- end }}
261+
262+
{{- else if (include "kafka.externalClient.tlsEncryption" .) }}
263+
264+
You need to configure your Kafka client to access using TLS authentication. To do so, you need to create the 'client.properties' configuration file with the content below:
265+
266+
security.protocol={{ $externalClientProtocol }}
267+
ssl.truststore.type={{ upper .Values.auth.tls.type }}
268+
{{- if eq .Values.auth.tls.type "jks" }}
269+
ssl.truststore.location=/tmp/kafka.truststore.{{ .Values.auth.tls.type }}
270+
{{- if not (empty $tlsPassword) }}
271+
ssl.truststore.password={{ $tlsPassword }}
272+
{{- end }}
273+
{{- else if eq .Values.auth.tls.type "pem" }}
274+
ssl.truststore.certificates=-----BEGIN CERTIFICATE----- \
275+
... \
276+
-----END CERTIFICATE-----
277+
{{- end }}
278+
{{- if eq .Values.auth.externalClientProtocol "mtls" }}
279+
ssl.keystore.type={{ upper .Values.auth.tls.type }}
280+
{{- if eq .Values.auth.tls.type "jks" }}
281+
ssl.keystore.location=/tmp/client.keystore.jks
282+
{{- if not (empty $tlsPassword) }}
283+
ssl.keystore.password={{ $tlsPassword }}
284+
{{- end }}
285+
{{- else if eq .Values.auth.tls.type "pem" }}
286+
ssl.keystore.certificate.chain=-----BEGIN CERTIFICATE----- \
287+
... \
288+
-----END CERTIFICATE-----
289+
ssl.keystore.key=-----BEGIN ENCRYPTED PRIVATE KEY----- \
290+
... \
291+
-----END ENCRYPTED PRIVATE KEY-----
292+
{{- end }}
293+
{{- end }}
294+
{{- if eq $tlsEndpointIdentificationAlgorithm "" }}
295+
ssl.endpoint.identification.algorithm=
296+
{{- end }}
297+
298+
{{- end }}
299+
{{- end }}
300+
{{- end }}
301+
{{- end }}
302+
303+
{{- include "kafka.checkRollingTags" . }}
304+
{{- include "kafka.validateValues" . }}

0 commit comments

Comments
 (0)