Skip to content

Commit 74ddec8

Browse files
jianxiaoguojianxiaoguo
committed
feat(addons): add airflow
1 parent 7aec8be commit 74ddec8

49 files changed

Lines changed: 5461 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎addons/airflow/2/chart/airflow/.helmignore‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*~
18+
# Various IDEs
19+
.project
20+
.idea/
21+
*.tmproj

‎addons/airflow/2/chart/airflow/Chart.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright Drycc Community.
2+
# SPDX-License-Identifier: APACHE-2.0
3+
4+
annotations:
5+
category: WorkFlow
6+
licenses: Apache-2.0
7+
apiVersion: v2
8+
appVersion: "2.8.0"
9+
dependencies:
10+
- name: common
11+
repository: oci://registry.drycc.cc/charts
12+
version: ~1.1.3
13+
description: Apache Airflow is a tool to express and execute workflows as directed acyclic graphs (DAGs). It includes utilities to schedule tasks, monitor task progress and handle task dependencies.
14+
home: https://bitnami.com
15+
icon: https://bitnami.com/assets/stacks/airflow/img/airflow-stack-220x234.png
16+
keywords:
17+
- apache
18+
- airflow
19+
- workflow
20+
- dag
21+
maintainers:
22+
- name: Drycc Community.
23+
url: https://github.com/bitnami/charts
24+
name: airflow
25+
sources:
26+
- https://github.com/bitnami/charts/tree/main/bitnami/airflow
27+
version: 16.1.7

‎addons/airflow/2/chart/airflow/README.md‎

Lines changed: 732 additions & 0 deletions
Large diffs are not rendered by default.

‎addons/airflow/2/chart/airflow/templates/NOTES.txt‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
CHART NAME: {{ .Chart.Name }}
2+
CHART VERSION: {{ .Chart.Version }}
3+
APP VERSION: {{ .Chart.AppVersion }}
4+
5+
{{- $releaseNamespace := .Release.Namespace }}
6+
{{- $clusterDomain := .Values.clusterDomain }}
7+
{{- $fullname := include "common.names.fullname" . }}
8+
{{- $airflowSecretName := include "airflow.secretName" . }}
9+
{{- $baseUrl := (include "airflow.baseUrl" .) }}
10+
11+
** Please be patient while the chart is being deployed **
12+
13+
{{- if .Values.diagnosticMode.enabled }}
14+
15+
The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with:
16+
17+
command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }}
18+
args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }}
19+
20+
Get the list of pods by executing:
21+
22+
kubectl get pods --namespace {{ $releaseNamespace }} -l app.kubernetes.io/instance={{ .Release.Name }}
23+
24+
Access the pod you want to debug by executing
25+
26+
kubectl exec --namespace {{ $releaseNamespace }} -ti <NAME OF THE POD> -- bash
27+
28+
In order to replicate the container startup scripts execute this command:
29+
30+
/opt/drycc/scripts/airflow/entrypoint.sh /opt/drycc/scripts/airflow/run.sh
31+
32+
{{- else }}
33+
34+
{{- if not $baseUrl }}
35+
###############################################################################
36+
### ERROR: You did not provide an external URL in your 'helm install' call ###
37+
###############################################################################
38+
39+
This deployment will be incomplete until you configure Airflow with a resolvable
40+
host. To configure Airflow with the URL of your service:
41+
42+
1. Get the Airflow URL by running:
43+
44+
{{- if eq "NodePort" .Values.service.type }}
45+
46+
export AIRFLOW_HOST=$(kubectl get nodes --namespace {{ $releaseNamespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
47+
export AIRFLOW_PORT=$(kubectl get --namespace {{ $releaseNamespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ $fullname }})
48+
49+
{{- else if eq "LoadBalancer" .Values.service.type }}
50+
51+
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
52+
Watch the status with: 'kubectl get svc --namespace {{ $releaseNamespace }} -w {{ $fullname }}'
53+
54+
export AIRFLOW_HOST=$(kubectl get svc --namespace {{ $releaseNamespace }} {{ $fullname }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}")
55+
export AIRFLOW_PORT=80
56+
57+
{{- end }}
58+
59+
2. Complete your Airflow deployment by running:
60+
61+
{{ include "common.utils.secret.getvalue" (dict "secret" $airflowSecretName "field" "airflow-password" "context" $) }}
62+
{{ include "common.utils.secret.getvalue" (dict "secret" $airflowSecretName "field" "airflow-fernet-key" "context" $) }}
63+
{{ include "common.utils.secret.getvalue" (dict "secret" $airflowSecretName "field" "airflow-secret-key" "context" $) }}
64+
helm upgrade --namespace {{ $releaseNamespace }} {{ .Release.Name }} oci://registry-1.docker.io/bitnamicharts/{{ .Chart.Name }} \
65+
--set service.type={{ .Values.service.type }} \
66+
--set web.baseUrl=http://$AIRFLOW_HOST:$AIRFLOW_PORT \
67+
--set auth.password=$AIRFLOW_PASSWORD \
68+
--set auth.fernetKey=$AIRFLOW_FERNETKEY \
69+
--set auth.secretKey=$AIRFLOW_SECRETKEY
70+
71+
{{- else }}
72+
73+
Airflow can be accessed via port {{ .Values.service.ports.http }} on the following DNS name from within your cluster:
74+
75+
{{ printf "%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain }}
76+
77+
To connect to Airflow from outside the cluster, perform the following steps:
78+
79+
{{- if .Values.ingress.enabled }}
80+
81+
1. Get the Airflow URL and associate its hostname to your cluster external IP:
82+
83+
export CLUSTER_IP=$(minikube ip) # On Minikube. Use: `kubectl cluster-info` on others K8s clusters
84+
echo "Airflow URL: http{{ if .Values.ingress.tls }}s{{ end }}://{{ .Values.ingress.hostname }}"
85+
echo "$CLUSTER_IP {{ .Values.ingress.hostname }}" | sudo tee -a /etc/hosts
86+
87+
{{- else if eq .Values.service.type "ClusterIP" }}
88+
89+
1. Create a port-forward to the service:
90+
91+
kubectl port-forward --namespace {{ $releaseNamespace }} svc/{{ $fullname }} {{ .Values.service.ports.http }}:{{ .Values.service.ports.http }} &
92+
echo "Airflow URL: http://127.0.0.1:{{ .Values.service.ports.http }}"
93+
94+
{{- else if eq .Values.service.type "NodePort" }}
95+
96+
1. Obtain the NodePort IP and port:
97+
98+
export NODE_IP=$(kubectl get nodes --namespace {{ $releaseNamespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
99+
export NODE_PORT=$(kubectl get svc --namespace {{ $releaseNamespace }} {{ $fullname }} -o jsonpath="{.spec.ports[0].nodePort}")
100+
echo "Airflow URL: http://${NODE_IP}:$NODE_PORT"
101+
102+
{{- else if eq .Values.service.type "LoadBalancer" }}
103+
104+
1. Obtain the LoadBalancer IP:
105+
106+
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
107+
Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -w {{ $fullname }}'
108+
109+
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ $fullname }} --template "{{ "{{ range (index .status.loadBalancer.ingress 0) }}{{ . }}{{ end }}" }}")
110+
echo "Airflow URL: http://${SERVICE_IP}:{{ .Values.service.ports.http }}"
111+
112+
{{- end }}
113+
114+
2. Open a browser and access Airflow using the obtained URL.
115+
116+
3. Get your Airflow login credentials by running:
117+
118+
{{ include "common.utils.secret.getvalue" (dict "secret" $airflowSecretName "field" "airflow-password" "context" $) }}
119+
echo User: {{ .Values.auth.username }}
120+
echo Password: $AIRFLOW_PASSWORD
121+
122+
{{- end }}
123+
{{- end }}
124+
125+
{{ include "airflow.validateValues" . }}
126+
{{ include "airflow.checkRollingTags" . }}

‎addons/airflow/2/chart/airflow/templates/_git_helpers.tpl‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
{{/*
2+
Copyright Drycc Community.
3+
SPDX-License-Identifier: APACHE-2.0
4+
*/}}
5+
6+
{{/* vim: set filetype=mustache: */}}
7+
{{/*
8+
Returns the name that will identify the repository internally and it will be used to create folders or
9+
volume names
10+
*/}}
11+
{{- define "airflow.git.repository.name" -}}
12+
{{- $defaultName := regexFind "/.*$" .repository | replace "//" "" | replace "/" "-" | replace "." "-" -}}
13+
{{- .name | default $defaultName | kebabcase -}}
14+
{{- end -}}
15+
16+
{{/*
17+
Returns the volume mounts that will be used by git containers (clone and sync)
18+
*/}}
19+
{{- define "airflow.git.volumeMounts" -}}
20+
{{- if .Values.git.dags.enabled }}
21+
- name: git-cloned-dags
22+
mountPath: /dags
23+
{{- end }}
24+
{{- if .Values.git.plugins.enabled }}
25+
- name: git-cloned-plugins
26+
mountPath: /plugins
27+
{{- end }}
28+
{{- end -}}
29+
30+
{{/*
31+
Returns the volume mounts that will be used by the main container
32+
*/}}
33+
{{- define "airflow.git.maincontainer.volumeMounts" -}}
34+
{{- if .Values.git.dags.enabled }}
35+
{{- range .Values.git.dags.repositories }}
36+
- name: git-cloned-dags
37+
mountPath: /opt/drycc/airflow/dags/git_{{ include "airflow.git.repository.name" . }}
38+
{{- if .path }}
39+
subPath: {{ include "airflow.git.repository.name" . }}/{{ .path }}
40+
{{- else }}
41+
subPath: {{ include "airflow.git.repository.name" . }}
42+
{{- end }}
43+
{{- end }}
44+
{{- end }}
45+
{{- if .Values.git.plugins.enabled }}
46+
{{- range .Values.git.plugins.repositories }}
47+
- name: git-cloned-plugins
48+
mountPath: /opt/drycc/airflow/plugins/git_{{ include "airflow.git.repository.name" . }}
49+
{{- if .path }}
50+
subPath: {{ include "airflow.git.repository.name" . }}/{{ .path }}
51+
{{- else }}
52+
subPath: {{ include "airflow.git.repository.name" . }}
53+
{{- end }}
54+
{{- end }}
55+
{{- end }}
56+
{{- end -}}
57+
58+
{{/*
59+
Returns the volumes that will be attached to the workload resources (deployment, statefulset, etc)
60+
*/}}
61+
{{- define "airflow.git.volumes" -}}
62+
{{- if .Values.git.dags.enabled }}
63+
- name: git-cloned-dags
64+
emptyDir: {}
65+
{{- end }}
66+
{{- if .Values.git.plugins.enabled }}
67+
- name: git-cloned-plugins
68+
emptyDir: {}
69+
{{- end }}
70+
{{- end -}}
71+
72+
{{/*
73+
Returns the init container that will clone repositories files from a given list of git repositories
74+
Usage:
75+
{{ include "airflow.git.containers.clone" ( dict "securityContext" .Values.path.to.the.component.securityContext "context" $ ) }}
76+
*/}}
77+
{{- define "airflow.git.containers.clone" -}}
78+
{{- if or .context.Values.git.dags.enabled .context.Values.git.plugins.enabled }}
79+
- name: clone-repositories
80+
image: {{ include "git.image" .context | quote }}
81+
imagePullPolicy: {{ .context.Values.git.image.pullPolicy | quote }}
82+
{{- if .securityContext.enabled }}
83+
securityContext: {{- omit .securityContext "enabled" | toYaml | nindent 4 }}
84+
{{- end }}
85+
{{- if .context.Values.git.clone.resources }}
86+
resources: {{- toYaml .context.Values.git.clone.resources | nindent 4 }}
87+
{{- end }}
88+
{{- if .context.Values.git.clone.command }}
89+
command: {{- include "common.tplvalues.render" (dict "value" .context.Values.git.clone.command "context" .context) | nindent 4 }}
90+
{{- else }}
91+
command:
92+
- /bin/bash
93+
{{- end }}
94+
{{- if .context.Values.git.clone.args }}
95+
args: {{- include "common.tplvalues.render" (dict "value" .context.Values.git.clone.args "context" .context) | nindent 4 }}
96+
{{- else }}
97+
args:
98+
- -ec
99+
- |
100+
. /opt/drycc/scripts/libfs.sh
101+
{{- if .context.Values.git.dags.enabled }}
102+
{{- range .context.Values.git.dags.repositories }}
103+
is_dir_empty "/dags/{{ include "airflow.git.repository.name" . }}" && git clone {{ .repository }} --branch {{ .branch }} /dags/{{ include "airflow.git.repository.name" . }}
104+
{{- end }}
105+
{{- end }}
106+
{{- if .context.Values.git.plugins.enabled }}
107+
{{- range .context.Values.git.plugins.repositories }}
108+
is_dir_empty "/plugins/{{ include "airflow.git.repository.name" . }}" && git clone {{ .repository }} --branch {{ .branch }} /plugins/{{ include "airflow.git.repository.name" . }}
109+
{{- end }}
110+
{{- end }}
111+
{{- end }}
112+
volumeMounts:
113+
{{- include "airflow.git.volumeMounts" .context | trim | nindent 4 }}
114+
{{- if .context.Values.git.clone.extraVolumeMounts }}
115+
{{- include "common.tplvalues.render" (dict "value" .context.Values.git.clone.extraVolumeMounts "context" .context) | nindent 4 }}
116+
{{- end }}
117+
{{- if .context.Values.git.clone.extraEnvVars }}
118+
env: {{- include "common.tplvalues.render" (dict "value" .context.Values.git.clone.extraEnvVars "context" .context) | nindent 4 }}
119+
{{- end }}
120+
{{- if or .context.Values.git.clone.extraEnvVarsCM .context.Values.git.clone.extraEnvVarsSecret }}
121+
envFrom:
122+
{{- if .context.Values.git.clone.extraEnvVarsCM }}
123+
- configMapRef:
124+
name: {{ .context.Values.git.clone.extraEnvVarsCM }}
125+
{{- end }}
126+
{{- if .context.Values.git.clone.extraEnvVarsSecret }}
127+
- secretRef:
128+
name: {{ .context.Values.git.clone.extraEnvVarsSecret }}
129+
{{- end }}
130+
{{- end }}
131+
{{- end }}
132+
{{- end -}}
133+
134+
{{/*
135+
Returns the container that will pull and sync repositories files from a given list of git repositories
136+
Usage:
137+
{{ include "airflow.git.containers.sync" ( dict "securityContext" .Values.path.to.the.component.securityContext "context" $ ) }}
138+
*/}}
139+
{{- define "airflow.git.containers.sync" -}}
140+
{{- if or .context.Values.git.dags.enabled .context.Values.git.plugins.enabled }}
141+
- name: sync-repositories
142+
image: {{ include "git.image" .context | quote }}
143+
imagePullPolicy: {{ .context.Values.git.image.pullPolicy | quote }}
144+
{{- if .securityContext.enabled }}
145+
securityContext: {{- omit .securityContext "enabled" | toYaml | nindent 4 }}
146+
{{- end }}
147+
{{- if .context.Values.git.sync.resources }}
148+
resources: {{- toYaml .context.Values.git.sync.resources | nindent 4 }}
149+
{{- end }}
150+
{{- if .context.Values.git.sync.command }}
151+
command: {{- include "common.tplvalues.render" (dict "value" .context.Values.git.sync.command "context" .context) | nindent 4 }}
152+
{{- else }}
153+
command:
154+
- /bin/bash
155+
{{- end }}
156+
{{- if .context.Values.git.sync.args }}
157+
args: {{- include "common.tplvalues.render" (dict "value" .context.Values.git.sync.args "context" .context) | nindent 4 }}
158+
{{- else }}
159+
args:
160+
- -ec
161+
- |
162+
while true; do
163+
{{- if .context.Values.git.dags.enabled }}
164+
{{- range .context.Values.git.dags.repositories }}
165+
cd /dags/{{ include "airflow.git.repository.name" . }} && git pull origin {{ .branch }} || true
166+
{{- end }}
167+
{{- end }}
168+
{{- if .context.Values.git.plugins.enabled }}
169+
{{- range .context.Values.git.plugins.repositories }}
170+
cd /plugins/{{ include "airflow.git.repository.name" . }} && git pull origin {{ .branch }} || true
171+
{{- end }}
172+
{{- end }}
173+
sleep {{ default "60" .context.Values.git.sync.interval }}
174+
done
175+
{{- end }}
176+
volumeMounts:
177+
{{- include "airflow.git.volumeMounts" .context | trim | nindent 4 }}
178+
{{- if .context.Values.git.sync.extraVolumeMounts }}
179+
{{- include "common.tplvalues.render" (dict "value" .context.Values.git.sync.extraVolumeMounts "context" .context) | nindent 4 }}
180+
{{- end }}
181+
{{- if .context.Values.git.sync.extraEnvVars }}
182+
env: {{- include "common.tplvalues.render" (dict "value" .context.Values.git.sync.extraEnvVars "context" .context) | nindent 4 }}
183+
{{- end }}
184+
{{- if or .context.Values.git.sync.extraEnvVarsCM .context.Values.git.sync.extraEnvVarsSecret }}
185+
envFrom:
186+
{{- if .context.Values.git.sync.extraEnvVarsCM }}
187+
- configMapRef:
188+
name: {{ .context.Values.git.sync.extraEnvVarsCM }}
189+
{{- end }}
190+
{{- if .context.Values.git.sync.extraEnvVarsSecret }}
191+
- secretRef:
192+
name: {{ .context.Values.git.sync.extraEnvVarsSecret }}
193+
{{- end }}
194+
{{- end }}
195+
{{- end }}
196+
{{- end -}}

0 commit comments

Comments
 (0)