chore(airflow): custom pip and requirement config

Author: jianxiaoguo

addons/airflow/2/chart/airflow/templates/worker/configmap.yaml

@@ -0,0 +1,25 @@
+{{- /*
+Copyright Drycc Community.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{- if or .Values.worker.requirements .Values.worker.pip }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "common.names.fullname" . }}-worker
+  namespace: {{ .Release.Namespace | quote }}
+  labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }}
+  {{- end }}
+data:
+  {{- if .Values.worker.requirements }}
+  requirements.txt: |-
+    {{- include "common.tplvalues.render" ( dict "value" .Values.worker.requirements "context" $ ) | nindent 4 }}
+  {{- end }}
+  {{- if .Values.worker.pip }}
+  pip.conf: |-
+    {{- include "common.tplvalues.render" ( dict "value" .Values.worker.pip "context" $ ) | nindent 4 }}
+  {{- end }}
+{{- end }}

addons/airflow/2/chart/airflow/templates/worker/statefulset.yaml

@@ -195,6 +195,16 @@ spec:
               mountPath: /opt/drycc/airflow/airflow.cfg
               subPath: airflow.cfg
             {{- end }}
+            {{- if .Values.worker.requirements }}
+            - name: worker-config
+              mountPath: /opt/drycc/python/requirements.txt
+              subPath: requirements.txt
+            {{- end }}
+            {{- if .Values.worker.pip }}
+            - name: worker-config
+              mountPath: /opt/drycc/.config/pip/pip.conf
+              subPath: pip.conf
+            {{- end }}
             {{- include "airflow.git.maincontainer.volumeMounts" . | trim | nindent 12 }}
             {{- if .Values.extraVolumeMounts }}
             {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
@@ -221,6 +231,11 @@ spec:
           configMap:
             name: {{ include "airflow.configMapName"  . }}
         {{- end }}
+        {{- if or .Values.worker.requirements .Values.worker.pip }}
+        - name: worker-config
+          configMap:
+            name: {{ include "common.names.fullname" . }}-worker
+        {{- end }}
         {{- if .Values.extraVolumes }}
         {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumes "context" $) | nindent 8 }}
         {{- end }}

addons/airflow/2/chart/airflow/values.yaml

@@ -706,6 +706,20 @@ worker:
   ## @param worker.extraEnvVarsSecrets List of secrets with extra environment variables for Airflow worker pods
   ##
   extraEnvVarsSecrets: []
+  ## @param requirements Specify content for Airflow worker requirements.txt file
+  ## e.g:
+  ## requirements: |-
+  ##   pylint==2.13.9
+  ##   ...
+  ##
+  requirements: ""
+  ## @param pip.conf Specify content for Airflow worker pip.conf file
+  ## e.g:
+  ## pip: |-
+  ##  [global]
+  ##   ...
+  ##
+  pip: ""
   ## @param worker.containerPorts.http Airflow worker HTTP container port
   ##
   containerPorts:

addons/airflow/2/meta.yaml

@@ -28,9 +28,9 @@ allow_parameters:
   required: false
   description: "metrics enable or not config for values.yaml"
 - name: "externalDatabase"
-  required: false
+  required: true
   description: "externalDatabase config for values.yaml"
 - name: "externalRedis"
-  required: false
+  required: true
   description: "externalRedis config for values.yaml"
 archive: false