feat(drycc-addons): add spark

Author: jianxiaoguo

addons/index.yaml

@@ -23,3 +23,6 @@ entries:
   zookeeper:
     - version: 3.9
       description: "Apache ZooKeeper is an effort to develop and maintain an open-source server which enables highly reliable distributed coordination."
+  spark:
+    - version: 3.4
+      description: "Apache Spark is a multi-language engine for executing data engineering, data science, and machine learning on single-node machines or clusters."

addons/minio/2023/chart/minio/values.yaml

@@ -843,7 +843,7 @@ networkPolicy:
   ## listening on. When true, MinIO® will accept connections from any source (with the correct destination port).
   ##
   allowExternal: false
-  allowLocalNamespace: true
+  allowCurrentNamespace: true
   allowNamespaces: []
   ## @param networkPolicy.extraFromClauses Allows to add extra 'from' clauses to the NetworkPolicy
   extraFromClauses: {}

addons/postgresql/15/chart/postgresql/values.yaml

@@ -955,7 +955,7 @@ networkPolicy:
   ## @param networkPolicy.enabled Enable network policies
   ##
   enabled: true
-  allowLocalNamespace: true
+  allowCurrentNamespace: true
   allowNamespaces: []
   ## @param networkPolicy.metrics.enabled Enable network policies for metrics (prometheus)
   ## @param networkPolicy.metrics.namespaceSelector [object] Monitoring namespace selector labels. These labels will be used to identify the prometheus' namespace.

addons/spark/3.4/chart/spark/Chart.yaml

@@ -0,0 +1,28 @@
+# Copyright VMware, Inc.
+# SPDX-License-Identifier: APACHE-2.0
+
+annotations:
+  category: Infrastructure
+  licenses: Apache-2.0
+  images: |
+    - name: spark
+      image: registry.drycc.cc/drycc-addons/spark:3.4
+apiVersion: v2
+appVersion: 3.4.1
+dependencies:
+  - name: common
+    repository: oci://registry.drycc.cc/charts
+    version: ~1.1.3
+description: Apache Spark is a high-performance engine for large-scale computing tasks, such as data processing, machine learning and real-time data streaming. It includes APIs for Java, Python, Scala and R.
+home: https://bitnami.com
+icon: https://bitnami.com/assets/stacks/spark/img/spark-stack-220x234.png
+keywords:
+  - apache
+  - spark
+maintainers:
+  - name: VMware, Inc.
+    url: https://github.com/bitnami/charts
+name: spark
+sources:
+  - https://github.com/bitnami/charts/tree/main/bitnami/spark
+version: 7.2.1

addons/spark/3.4/chart/spark/README.md

@@ -0,0 +1,102 @@
+CHART NAME: {{ .Chart.Name }}
+CHART VERSION: {{ .Chart.Version }}
+APP VERSION: {{ .Chart.AppVersion }}
+
+** Please be patient while the chart is being deployed **
+
+{{- if .Values.diagnosticMode.enabled }}
+The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with:
+
+  command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }}
+  args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }}
+
+Get the list of pods by executing:
+
+  kubectl get pods --namespace {{ include "common.names.namespace" . }} -l app.kubernetes.io/instance={{ .Release.Name }}
+
+Access the pod you want to debug by executing
+
+  kubectl exec --namespace {{ include "common.names.namespace" . }} -ti <NAME OF THE POD> -- bash
+
+In order to replicate the container startup scripts execute this command:
+
+    /opt/bitnami/scripts/spark/entrypoint.sh /opt/bitnami/scripts/spark/run.sh
+
+{{- else }}
+
+1. Get the Spark master WebUI URL by running these commands:
+{{- if .Values.ingress.enabled }}
+
+  export HOSTNAME=$(kubectl get ingress --namespace {{ include "common.names.namespace" . }} {{ printf "%s-ingress" (include "common.names.fullname" .) }} -o jsonpath='{.spec.rules[0].host}')
+  echo "Spark-master URL: http://$HOSTNAME/"
+
+{{- else -}}
+{{- if contains "NodePort" .Values.service.type }}
+
+  export NODE_PORT=$(kubectl get --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.spec.ports[?(@.name=='http')].nodePort}" services {{ printf "%s-master-svc" (include "common.names.fullname" .) }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+
+{{- else if contains "LoadBalancer" .Values.service.type }}
+
+    NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+    You can watch the status of by running 'kubectl get --namespace {{ include "common.names.namespace" . }} svc -w {{ printf "%s-master-svc" (include "common.names.fullname" .) }}'
+
+  export SERVICE_IP=$(kubectl get --namespace {{ include "common.names.namespace" . }} svc {{ printf "%s-master-svc" (include "common.names.fullname" .) }} -o jsonpath="{.status.loadBalancer.ingress[0]['ip', 'hostname'] }")
+  echo http://$SERVICE_IP:{{ .Values.service.ports.http }}
+
+{{- else if contains "ClusterIP" .Values.service.type }}
+
+  kubectl port-forward --namespace {{ include "common.names.namespace" . }} svc/{{ printf "%s-master-svc" (include "common.names.fullname" .) }} {{ default "80" .Values.service.ports.http }}:{{ default "80" .Values.service.ports.http }}
+  echo "Visit http://127.0.0.1:{{ .Values.service.ports.http }} to use your application"
+
+{{- end }}
+{{- end }}
+
+2. Submit an application to the cluster:
+
+  To submit an application to the cluster the spark-submit script must be used. That script can be
+  obtained at https://github.com/apache/spark/tree/master/bin. Also you can use kubectl run.
+
+{{- if or (eq "NodePort" .Values.service.type) (eq "LoadBalancer" .Values.service.type) }}
+
+  Run the commands below to obtain the master IP and submit your application.
+{{- end }}
+
+  export EXAMPLE_JAR=$(kubectl exec -ti --namespace {{ include "common.names.namespace" . }} {{ printf "%s-worker-0" (include "common.names.fullname" .) }} -- find examples/jars/ -name 'spark-example*\.jar' | tr -d '\r')
+{{- if eq "NodePort" .Values.service.type }}
+  export SUBMIT_PORT=$(kubectl get --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.spec.ports[?(@.name=='cluster')].nodePort}" services {{ printf "%s-master-svc" (include "common.names.fullname" .) }})
+  export SUBMIT_IP=$(kubectl get nodes --namespace {{ include "common.names.namespace" . }} -o jsonpath="{.items[0].status.addresses[0].address}")
+
+  kubectl run --namespace {{ include "common.names.namespace" . }} {{ printf "%s-client" (include "common.names.fullname" .) }} --rm --tty -i --restart='Never' \
+    --image {{ template "spark.image" . }} \
+    -- spark-submit --master spark://$SUBMIT_IP:$SUBMIT_PORT \
+    --class org.apache.spark.examples.SparkPi \
+    --deploy-mode cluster \
+    $EXAMPLE_JAR 1000
+
+{{- else if eq "LoadBalancer" .Values.service.type }}
+  export SUBMIT_IP=$(kubectl get --namespace {{ include "common.names.namespace" . }} svc {{ printf "%s-master-svc" (include "common.names.fullname" .) }} -o jsonpath="{.status.loadBalancer.ingress[0]['ip', 'hostname'] }")
+
+  kubectl run --namespace {{ include "common.names.namespace" . }} {{ printf "%s-client" (include "common.names.fullname" .) }} --rm --tty -i --restart='Never' \
+    --image {{ template "spark.image" . }} \
+    -- spark-submit --master spark://$SUBMIT_IP:{{ .Values.service.ports.cluster }} \
+    --deploy-mode cluster \
+    --class org.apache.spark.examples.SparkPi \
+    $EXAMPLE_JAR 1000
+
+{{- else }}
+
+  kubectl exec -ti --namespace {{ include "common.names.namespace" . }} {{ printf "%s-worker-0" (include "common.names.fullname" .) }} -- spark-submit --master spark://{{ printf "%s-master-svc" (include "common.names.fullname" .) }}:{{ .Values.service.ports.cluster }} \
+    --class org.apache.spark.examples.SparkPi \
+    $EXAMPLE_JAR 5
+
+** IMPORTANT: When submit an application from outside the cluster service type should be set to the NodePort or LoadBalancer. **
+
+{{- end }}
+
+** IMPORTANT: When submit an application the --master parameter should be set to the service IP, if not, the application will not resolve the master. **
+{{- end }}
+
+{{ include "spark.checkRollingTags" . }}
+{{ include "spark.validateValues" . }}

addons/spark/3.4/chart/spark/templates/NOTES.txt

@@ -0,0 +1,132 @@
+{{/*
+Copyright VMware, Inc.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{- /* vim: set filetype=mustache: */}}
+
+{{/*
+Return the proper Spark image name
+*/}}
+{{- define "spark.image" -}}
+{{- include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) -}}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names
+*/}}
+{{- define "spark.imagePullSecrets" -}}
+{{- include "common.images.pullSecrets" (dict "images" (list .Values.image) "global" .Values.global) -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "spark.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "common.names.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/* Get the secret for passwords */}}
+{{- define "spark.passwordsSecretName" -}}
+{{- if .Values.security.passwordsSecretName -}}
+  {{- printf "%s" .Values.security.passwordsSecretName -}}
+{{- else }}
+  {{- printf "%s-secret" (include "common.names.fullname" .) -}}
+{{- end }}
+{{- end -}}
+
+{{/*
+Return the secret containing Spark TLS certificates
+*/}}
+{{- define "spark.tlsSecretName" -}}
+{{- $secretName := coalesce .Values.security.ssl.existingSecret .Values.security.certificatesSecretName -}}
+{{- if $secretName -}}
+    {{- printf "%s" (tpl $secretName $) -}}
+{{- else -}}
+    {{- printf "%s-crt" (include "common.names.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a TLS secret object should be created
+*/}}
+{{- define "spark.createTlsSecret" -}}
+{{- if and .Values.security.ssl.autoGenerated .Values.security.ssl.enabled (not .Values.security.ssl.existingSecret) (not .Values.security.certificatesSecretName) }}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Check if there are rolling tags in the images */}}
+{{- define "spark.checkRollingTags" -}}
+{{- include "common.warnings.rollingTag" .Values.image -}}
+{{- end -}}
+
+{{/* Validate values of Spark - Incorrect extra volume settings */}}
+{{- define "spark.validateValues.extraVolumes" -}}
+{{- if and (.Values.worker.extraVolumes) (not .Values.worker.extraVolumeMounts) -}}
+spark: missing-worker-extra-volume-mounts
+    You specified worker extra volumes but no mount points for them. Please set
+    the extraVolumeMounts value
+{{- end -}}
+{{- end -}}
+
+{/* Validate values of Spark - number of workers must be greater than 0 */}}
+{{- define "spark.validateValues.workerCount" -}}
+{{- $replicaCount := int .Values.worker.replicaCount }}
+{{- if lt $replicaCount 1 -}}
+spark: workerCount
+    Worker replicas must be greater than 0!!
+    Please set a valid worker count size (--set worker.replicaCount=X)
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Spark - Security SSL enabled */}}
+{{- define "spark.validateValues.security.ssl" -}}
+{{- if and .Values.security.ssl.enabled (not .Values.security.ssl.autoGenerated) (not .Values.security.ssl.existingSecret) (not .Values.security.certificatesSecretName) }}
+spark: security.ssl.enabled
+    In order to enable Security SSL, you also need to provide
+    an existing secret containing the Keystore and Truststore or
+    enable auto-generated certificates.
+{{- end -}}
+{{- end -}}
+
+{{/*
+Compile all warnings into a single message, and call fail.
+*/}}
+{{- define "spark.validateValues" -}}
+{{- $messages := list -}}
+{{- $messages := append $messages (include "spark.validateValues.extraVolumes" .) -}}
+{{- $messages := append $messages (include "spark.validateValues.workerCount" .) -}}
+{{- $messages := append $messages (include "spark.validateValues.security.ssl" .) -}}
+{{- $messages := without $messages "" -}}
+{{- $message := join "\n" $messages -}}
+
+{{- if $message -}}
+{{-   printf "\nVALUES VALIDATION:\n%s" $message | fail -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Get the initialization scripts volume name.
+*/}}
+{{- define "spark.initScripts" -}}
+{{- printf "%s-init-scripts" (include "common.names.fullname" .) -}}
+{{- end -}}
+
+{{/*
+Get the initialization scripts ConfigMap name.
+*/}}
+{{- define "spark.initScriptsCM" -}}
+{{- printf "%s" .Values.initScriptsCM -}}
+{{- end -}}
+
+{{/*
+Get the initialization scripts Secret name.
+*/}}
+{{- define "spark.initScriptsSecret" -}}
+{{- printf "%s" .Values.initScriptsSecret -}}
+{{- end -}}

addons/spark/3.4/chart/spark/templates/_helpers.tpl

@@ -0,0 +1,9 @@
+{{- /*
+Copyright VMware, Inc.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{- range .Values.extraDeploy }}
+---
+{{ include "common.tplvalues.render" (dict "value" . "context" $) }}
+{{- end }}

addons/spark/3.4/chart/spark/templates/extra-list.yaml

@@ -0,0 +1,19 @@
+{{- /*
+Copyright VMware, Inc.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ printf "%s-headless" (include "common.names.fullname" .) }}
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
+  {{- if or .Values.service.headless.annotations .Values.commonAnnotations }}
+  {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.service.headless.annotations .Values.commonAnnotations ) "context" . ) }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  selector: {{- include "common.labels.matchLabels" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}

addons/spark/3.4/chart/spark/templates/headless-svc.yaml

@@ -0,0 +1,49 @@
+{{- /*
+Copyright VMware, Inc.
+SPDX-License-Identifier: APACHE-2.0
+*/}}
+
+{{- if .Values.worker.autoscaling.enabled }}
+apiVersion: {{ include "common.capabilities.hpa.apiVersion" ( dict "context" $ ) }}
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ printf "%s-autoscaler" (include "common.names.fullname" .) }}
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
+    app.kubernetes.io/component: worker-autoscaler
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  scaleTargetRef:
+    apiVersion: {{ include "common.capabilities.statefulset.apiVersion" . }}
+    kind: statefulset
+    name: {{ printf "%s-worker" (include "common.names.fullname" .) }}
+  minReplicas: {{ .Values.worker.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.worker.autoscaling.targetMemory }}
+    - type: Resource
+      resource:
+        name: memory
+        {{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
+        targetAverageUtilization: {{ .Values.worker.autoscaling.targetMemory }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.worker.autoscaling.targetMemory }}
+        {{- end }}
+    {{- end }}
+    {{- if .Values.worker.autoscaling.targetCPU }}
+    - type: Resource
+      resource:
+        name: cpu
+        {{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) }}
+        targetAverageUtilization: {{ .Values.worker.autoscaling.targetCPU }}
+        {{- else }}
+        target:
+          type: Utilization
+          averageUtilization: {{ .Values.worker.autoscaling.targetCPU }}
+        {{- end }}
+    {{- end }}
+{{- end }}