Skip to content

Commit d9a162a

Browse files
jianxiaoguojianxiaoguo
committed
chore(addons): spark chart volume work dir
1 parent 4322e84 commit d9a162a

10 files changed

Lines changed: 282 additions & 9 deletions

File tree

‎addons/spark/3.4/chart/spark/templates/statefulset-master.yaml‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,8 @@ spec:
200200
- name: custom-init-scripts-secret
201201
mountPath: /docker-entrypoint-initdb.d/init-scripts-secret
202202
{{- end }}
203+
- name: work-data
204+
mountPath: /opt/drycc/spark/work
203205
env:
204206
- name: DRYCC_DEBUG
205207
value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
@@ -344,6 +346,43 @@ spec:
344346
secretName: {{ template ".Values.initScriptsSecret" . }}
345347
defaultMode: 0755
346348
{{- end }}
347-
{{- if .Values.master.extraVolumeClaimTemplates }}
348-
volumeClaimTemplates: {{- include "common.tplvalues.render" (dict "value" .Values.master.extraVolumeClaimTemplates "context" $) | nindent 8 }}
349+
{{- if not .Values.master.persistence.enabled }}
350+
- name: "work-data"
351+
emptyDir: {}
352+
{{- else if .Values.master.persistence.existingClaim }}
353+
- name: "work-data"
354+
persistentVolumeClaim:
355+
claimName: {{ .Values.master.persistence.existingClaim }}
356+
{{- else }}
357+
{{- if .Values.master.persistentVolumeClaimRetentionPolicy.enabled }}
358+
persistentVolumeClaimRetentionPolicy:
359+
whenDeleted: {{ .Values.master.persistentVolumeClaimRetentionPolicy.whenDeleted }}
360+
whenScaled: {{ .Values.master.persistentVolumeClaimRetentionPolicy.whenScaled }}
361+
{{- end }}
362+
volumeClaimTemplates:
363+
- metadata:
364+
name: "work-data"
365+
{{- if or .Values.master.persistence.annotations .Values.commonAnnotations }}
366+
{{- $claimAnnotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.master.persistence.annotations .Values.commonAnnotations ) "context" . ) }}
367+
annotations: {{- include "common.tplvalues.render" ( dict "value" $claimAnnotations "context" $) | nindent 4 }}
368+
{{- end }}
369+
{{- if .Values.commonLabels }}
370+
labels: {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 10 }}
371+
{{- end }}
372+
spec:
373+
accessModes:
374+
{{- range .Values.master.persistence.accessModes }}
375+
- {{ . | quote }}
376+
{{- end }}
377+
resources:
378+
requests:
379+
storage: {{ .Values.master.persistence.size | quote }}
380+
{{- if .Values.master.persistence.selector }}
381+
selector: {{- include "common.tplvalues.render" (dict "value" .Values.master.persistence.selector "context" $) | nindent 10 }}
382+
{{- else if .Values.master.persistence.existingVolume }}
383+
selector:
384+
matchLabels:
385+
volume: {{ .Values.master.persistence.existingVolume }}
386+
{{- end }}
387+
{{- include "common.storage.class" (dict "persistence" .Values.master.persistence "global" .Values.global) | nindent 8 }}
349388
{{- end }}

‎addons/spark/3.4/chart/spark/templates/statefulset-worker.yaml‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,8 @@ spec:
206206
- name: custom-init-scripts-secret
207207
mountPath: /docker-entrypoint-initdb.d/init-scripts-secret
208208
{{- end }}
209+
- name: work-data
210+
mountPath: /opt/drycc/spark/work
209211
env:
210212
- name: SPARK_MODE
211213
value: "worker"
@@ -368,6 +370,43 @@ spec:
368370
secretName: {{ template ".Values.initScriptsSecret" . }}
369371
defaultMode: 0755
370372
{{- end }}
371-
{{- if .Values.worker.extraVolumeClaimTemplates }}
372-
volumeClaimTemplates: {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumeClaimTemplates "context" $) | nindent 8 }}
373+
{{- if not .Values.worker.persistence.enabled }}
374+
- name: "work-data"
375+
emptyDir: {}
376+
{{- else if .Values.worker.persistence.existingClaim }}
377+
- name: "work-data"
378+
persistentVolumeClaim:
379+
claimName: {{ .Values.worker.persistence.existingClaim }}
380+
{{- else }}
381+
{{- if .Values.worker.persistentVolumeClaimRetentionPolicy.enabled }}
382+
persistentVolumeClaimRetentionPolicy:
383+
whenDeleted: {{ .Values.worker.persistentVolumeClaimRetentionPolicy.whenDeleted }}
384+
whenScaled: {{ .Values.worker.persistentVolumeClaimRetentionPolicy.whenScaled }}
385+
{{- end }}
386+
volumeClaimTemplates:
387+
- metadata:
388+
name: "work-data"
389+
{{- if or .Values.worker.persistence.annotations .Values.commonAnnotations }}
390+
{{- $claimAnnotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.worker.persistence.annotations .Values.commonAnnotations ) "context" . ) }}
391+
annotations: {{- include "common.tplvalues.render" ( dict "value" $claimAnnotations "context" $) | nindent 4 }}
392+
{{- end }}
393+
{{- if .Values.commonLabels }}
394+
labels: {{- include "common.tplvalues.render" (dict "value" .Values.commonLabels "context" $) | nindent 10 }}
395+
{{- end }}
396+
spec:
397+
accessModes:
398+
{{- range .Values.worker.persistence.accessModes }}
399+
- {{ . | quote }}
400+
{{- end }}
401+
resources:
402+
requests:
403+
storage: {{ .Values.worker.persistence.size | quote }}
404+
{{- if .Values.worker.persistence.selector }}
405+
selector: {{- include "common.tplvalues.render" (dict "value" .Values.worker.persistence.selector "context" $) | nindent 10 }}
406+
{{- else if .Values.worker.persistence.existingVolume }}
407+
selector:
408+
matchLabels:
409+
volume: {{ .Values.worker.persistence.existingVolume }}
410+
{{- end }}
411+
{{- include "common.storage.class" (dict "persistence" .Values.worker.persistence "global" .Values.global) | nindent 8 }}
373412
{{- end }}

‎addons/spark/3.4/chart/spark/values.yaml‎

Lines changed: 98 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,55 @@ master:
383383
## containerPort: 1234
384384
##
385385
initContainers: []
386-
386+
## Enable persistence using Persistent Volume Claims
387+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
388+
##
389+
persistence:
390+
## @param master.persistence.enabled Enable persistence using a `PersistentVolumeClaim`
391+
##
392+
enabled: false
393+
## @param master.persistence.storageClass Persistent Volume Storage Class
394+
## If defined, storageClassName: <storageClass>
395+
## If set to "-", storageClassName: "", which disables dynamic provisioning
396+
## If undefined (the default) or set to null, no storageClassName spec is
397+
## set, choosing the default provisioner. (gp2 on AWS, standard on
398+
## GKE, AWS & OpenStack)
399+
##
400+
storageClass: ""
401+
## @param master.persistence.existingClaim Existing Persistent Volume Claim
402+
## then accept the value as an existing Persistent Volume Claim to which
403+
## the container should be bound
404+
##
405+
existingClaim: ""
406+
## @param master.persistence.existingVolume Existing Persistent Volume for use as volume match label selector to the `volumeClaimTemplate`. Ignored when `master.persistence.selector` is set.
407+
##
408+
existingVolume: ""
409+
## @param master.persistence.selector Configure custom selector for existing Persistent Volume. Overwrites `master.persistence.existingVolume`
410+
## selector:
411+
## matchLabels:
412+
## volume:
413+
##
414+
selector: {}
415+
## @param master.persistence.annotations Persistent Volume Claim annotations
416+
##
417+
annotations: {}
418+
## @param master.persistence.accessModes Persistent Volume Access Modes
419+
##
420+
accessModes:
421+
- ReadWriteOnce
422+
## @param master.persistence.size Persistent Volume Size
423+
##
424+
size: 1Gi
425+
## persistentVolumeClaimRetentionPolicy
426+
## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
427+
## @param master.persistentVolumeClaimRetentionPolicy.enabled Controls if and how PVCs are deleted during the lifecycle of a StatefulSet
428+
## @param master.persistentVolumeClaimRetentionPolicy.whenScaled Volume retention behavior when the replica count of the StatefulSet is reduced
429+
## @param master.persistentVolumeClaimRetentionPolicy.whenDeleted Volume retention behavior that applies when the StatefulSet is deleted
430+
##
431+
persistentVolumeClaimRetentionPolicy:
432+
enabled: true
433+
whenScaled: Delete
434+
whenDeleted: Delete
387435
## @section Spark worker parameters
388436
##
389437

@@ -680,6 +728,55 @@ worker:
680728
maxReplicas: 5
681729
targetCPU: 50
682730
targetMemory: ""
731+
## Enable persistence using Persistent Volume Claims
732+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
733+
##
734+
persistence:
735+
## @param worker.persistence.enabled Enable persistence using a `PersistentVolumeClaim`
736+
##
737+
enabled: false
738+
## @param worker.persistence.storageClass Persistent Volume Storage Class
739+
## If defined, storageClassName: <storageClass>
740+
## If set to "-", storageClassName: "", which disables dynamic provisioning
741+
## If undefined (the default) or set to null, no storageClassName spec is
742+
## set, choosing the default provisioner. (gp2 on AWS, standard on
743+
## GKE, AWS & OpenStack)
744+
##
745+
storageClass: ""
746+
## @param worker.persistence.existingClaim Existing Persistent Volume Claim
747+
## then accept the value as an existing Persistent Volume Claim to which
748+
## the container should be bound
749+
##
750+
existingClaim: ""
751+
## @param worker.persistence.existingVolume Existing Persistent Volume for use as volume match label selector to the `volumeClaimTemplate`. Ignored when `worker.persistence.selector` is set.
752+
##
753+
existingVolume: ""
754+
## @param worker.persistence.selector Configure custom selector for existing Persistent Volume. Overwrites `worker.persistence.existingVolume`
755+
## selector:
756+
## matchLabels:
757+
## volume:
758+
##
759+
selector: {}
760+
## @param worker.persistence.annotations Persistent Volume Claim annotations
761+
##
762+
annotations: {}
763+
## @param worker.persistence.accessModes Persistent Volume Access Modes
764+
##
765+
accessModes:
766+
- ReadWriteOnce
767+
## @param worker.persistence.size Persistent Volume Size
768+
##
769+
size: 1Gi
770+
## persistentVolumeClaimRetentionPolicy
771+
## ref: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#persistentvolumeclaim-retention
772+
## @param worker.persistentVolumeClaimRetentionPolicy.enabled Controls if and how PVCs are deleted during the lifecycle of a StatefulSet
773+
## @param worker.persistentVolumeClaimRetentionPolicy.whenScaled Volume retention behavior when the replica count of the StatefulSet is reduced
774+
## @param worker.persistentVolumeClaimRetentionPolicy.whenDeleted Volume retention behavior that applies when the StatefulSet is deleted
775+
##
776+
persistentVolumeClaimRetentionPolicy:
777+
enabled: true
778+
whenScaled: Delete
779+
whenDeleted: Delete
683780

684781
## @section Security parameters
685782
##

‎addons/spark/3.4/meta.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,15 @@ allow_parameters:
2424
- name: "master.configOptions"
2525
required: false
2626
description: "master configOptions config for values.yaml"
27+
- name: "master.persistence"
28+
required: false
29+
description: "master persistence config for values.yaml"
2730
- name: "worker.configOptions"
2831
required: false
2932
description: "worker configOptions config for values.yaml"
33+
- name: "worker.persistence"
34+
required: false
35+
description: "worker persistence config for values.yaml"
3036
- name: "metrics.enable"
3137
required: false
3238
description: "metrics enable or not config for values.yaml"

‎addons/spark/3.4/plans/standard-16c32g5w/values.yaml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,14 @@ master:
3333
requests:
3434
cpu: 4000m
3535
memory: 16Gi
36-
36+
## Enable persistence using Persistent Volume Claims
37+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
38+
##
39+
persistence:
40+
enabled: true
41+
accessModes:
42+
- ReadWriteOnce
43+
size: 1Gi
3744
## @section Spark worker parameters
3845
##
3946

@@ -78,3 +85,11 @@ worker:
7885
requests:
7986
cpu: 4000m
8087
memory: 16Gi
88+
## Enable persistence using Persistent Volume Claims
89+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
90+
##
91+
persistence:
92+
enabled: true
93+
accessModes:
94+
- ReadWriteOnce
95+
size: 1Gi

‎addons/spark/3.4/plans/standard-1c2g2w/values.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@ master:
3333
requests:
3434
cpu: 500m
3535
memory: 1Gi
36+
## Enable persistence using Persistent Volume Claims
37+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
38+
##
39+
persistence:
40+
enabled: true
41+
accessModes:
42+
- ReadWriteOnce
43+
size: 1Gi
3644

3745
## @section Spark worker parameters
3846
##
@@ -78,3 +86,11 @@ worker:
7886
requests:
7987
cpu: 500m
8088
memory: 1Gi
89+
## Enable persistence using Persistent Volume Claims
90+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
91+
##
92+
persistence:
93+
enabled: true
94+
accessModes:
95+
- ReadWriteOnce
96+
size: 1Gi

‎addons/spark/3.4/plans/standard-24c48g5w/values.yaml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,14 @@ master:
3333
requests:
3434
cpu: 8000m
3535
memory: 24Gi
36-
36+
## Enable persistence using Persistent Volume Claims
37+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
38+
##
39+
persistence:
40+
enabled: true
41+
accessModes:
42+
- ReadWriteOnce
43+
size: 1Gi
3744
## @section Spark worker parameters
3845
##
3946

@@ -78,3 +85,11 @@ worker:
7885
requests:
7986
cpu: 8000m
8087
memory: 24Gi
88+
## Enable persistence using Persistent Volume Claims
89+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
90+
##
91+
persistence:
92+
enabled: true
93+
accessModes:
94+
- ReadWriteOnce
95+
size: 1Gi

‎addons/spark/3.4/plans/standard-2c4g2w/values.yaml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,14 @@ master:
3333
requests:
3434
cpu: 1000m
3535
memory: 2Gi
36-
36+
## Enable persistence using Persistent Volume Claims
37+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
38+
##
39+
persistence:
40+
enabled: true
41+
accessModes:
42+
- ReadWriteOnce
43+
size: 1Gi
3744
## @section Spark worker parameters
3845
##
3946

@@ -78,3 +85,11 @@ worker:
7885
requests:
7986
cpu: 1000m
8087
memory: 2Gi
88+
## Enable persistence using Persistent Volume Claims
89+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
90+
##
91+
persistence:
92+
enabled: true
93+
accessModes:
94+
- ReadWriteOnce
95+
size: 1Gi

‎addons/spark/3.4/plans/standard-4c8g3w/values.yaml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,14 @@ master:
3333
requests:
3434
cpu: 2000m
3535
memory: 4Gi
36-
36+
## Enable persistence using Persistent Volume Claims
37+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
38+
##
39+
persistence:
40+
enabled: true
41+
accessModes:
42+
- ReadWriteOnce
43+
size: 1Gi
3744
## @section Spark worker parameters
3845
##
3946

@@ -78,3 +85,11 @@ worker:
7885
requests:
7986
cpu: 2000m
8087
memory: 4Gi
88+
## Enable persistence using Persistent Volume Claims
89+
## ref: https://kubernetes.io/docs/user-guide/persistent-volumes/
90+
##
91+
persistence:
92+
enabled: true
93+
accessModes:
94+
- ReadWriteOnce
95+
size: 1Gi

0 commit comments

Comments
 (0)