chore(postgresql-cluster): Adjust backup strategy

Author: EamonZhang

addons/postgresql-cluster/15/chart/postgresql-cluster/README.md

@@ -74,7 +74,7 @@ backup:
 
 You can modify multiple content at once, there is no need to modify part of it each time.
 
-# Plans
+## Plans
 
 | Resource Specification | Cores | MEMORY | Storage SIZE |
 | :---: | :---: | :---: | :---: | 

addons/postgresql-cluster/15/chart/postgresql-cluster/templates/_helpers.tpl

@@ -136,43 +136,14 @@ Create patroni envs.
 {{- end -}}
 
 {{/*
-Create walg envs.
+Return true if a configmap object should be created for PG backup.
 */}}
-{{- define "backup.envs" }}
-{{- if .Values.backup.enabled }}
-- name: USE_WALG
-  value: {{ .Values.backup.enabled | quote }}
-{{- if .Values.backup.retainBackups }}
-- name: BACKUP_NUM_TO_RETAIN
-  value: {{ .Values.backup.retainBackups | quote}}
-{{- end }}
-{{- if .Values.backup.backupThresholdMegabytes }}
-- name: WALG_BACKUP_THRESHOLD_MEGABYTES
-  value: {{ .Values.backup.backupThresholdMegabytes | quote }}
-{{- end }}
-{{- if .Values.backup.backupThresholdPercentage }}
-- name: WALE_BACKUP_THRESHOLD_PERCENTAGE
-  value: {{ .Values.backup.backupThresholdPercentage | quote }}
-{{- end }}
-{{- if .Values.backup.s3.used }}
-- name: AWS_ACCESS_KEY_ID
-  value: {{ .Values.backup.s3.awsAccessKeyID | quote }}
-- name: AWS_SECRET_ACCESS_KEY
-  value: {{ .Values.backup.s3.awsSecretAccessKey | quote }}
-- name: WALG_S3_PREFIX
-  value: {{ .Values.backup.s3.walGS3Prefix | quote }}
-- name: AWS_ENDPOINT
-  value: {{ .Values.backup.s3.awsEndpoint | quote }}
-- name: AWS_S3_FORCE_PATH_STYLE
-  value: {{ .Values.backup.s3.awsS3ForcePathStyle | quote }}
-- name: AWS_REGION
-  value: {{ .Values.backup.s3.awsRegion | quote }}
-{{- end }}
-{{- else }}
-- name: USE_WALG
-  value: ""
-{{- end }}
-{{- end }}
+{{- define "backup.createConfigmap" -}}
+{{- if and .Values.backup.enabled }}
+    {{- true -}}
+{{- else -}}
+{{- end -}}
+{{- end -}}
 
 {{/*
 Generate random password 

addons/postgresql-cluster/15/chart/postgresql-cluster/templates/cm-backup.yaml

@@ -0,0 +1,22 @@
+{{- if (include "backup.createConfigmap" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "common.names.fullname" . }}-backup
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+    cluster-name: {{ template "patroni.fullname" . }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  backup.env: |-
+    {{- include "common.tplvalues.render" ( dict "value" .Values.backupEnv "context" $ ) | nindent 4 }}
+  # archive_command.sh: | -
+  #   #/bin/bash
+  #   source backup.env 
+  #   [ $USE_WALG ] ? sh /opt/drycc/postgresql/walbackup.sh %p : /bin/true
+{{- end -}}

addons/postgresql-cluster/15/chart/postgresql-cluster/templates/statefulset.yaml

@@ -56,9 +56,6 @@ spec:
         #   fsGroup: postgres
         env:
         {{- include "patroni.envs" . | indent 8 }} 
-        {{- if .Values.backup.enabled }}
-        {{- include "backup.envs" . | indent 8 }} 
-        {{- end }}
         {{- if .Values.env }}
         {{- range $key, $val := .Values.env }}
         - name: {{  $key | quote | upper }}
@@ -110,6 +107,9 @@ spec:
         - mountPath: "/opt/drycc/postgresql/config/"
           name: postgresql-config
           # readOnly: true
+        - mountPath: "/opt/drycc/postgresql/backup/"
+          name: postgresql-backup
+          # readOnly: true
         resources:
 {{ toYaml .Values.resources | indent 10 }}
       {{- if .Values.metrics.enabled }}
@@ -175,7 +175,6 @@ spec:
         - | 
           python3 /opt/drycc/postgresql/pgbackup.py 0.0.0.0 9000 
         env:
-        {{- include "backup.envs" . | indent 8 }} 
         - name: PGHOST
           value: localhost
         - name: PGPASSWORD
@@ -222,6 +221,9 @@ spec:
       - name: postgresql-config
         configMap:
           name: {{ template "common.names.fullname" . }}-postgresql
+      - name: archive-config
+        configMap:
+          name: {{ template "common.names.fullname" . }}-backup
       {{- if not .Values.persistentVolume.enabled }}
       - name: storage-volume
         emptyDir: {}

addons/postgresql-cluster/15/chart/postgresql-cluster/values.yaml

@@ -81,11 +81,7 @@ preInitScript: |
           track_commit_timestamp: "off"
           archive_mode: "on"
           archive_timeout: 300s
-          {{- if .Values.backup.enabled }}
-          archive_command: sh /opt/drycc/postgresql/walbackup.sh %p
-          {{- else }}
-          archive_command: /bin/true
-          {{- end }}
+          archive_command: sh /opt/drycc/postgresql/walbackup.sh %p %p
           # timescaledb.license: 'timescale'
           shared_preload_libraries: 'auto_explain,pg_stat_statements'
           log_destination: 'csvlog'
@@ -134,9 +130,20 @@ postInitScript: |
     echo "Skipping create admin user"
   fi
   psql -w -c  "CHECKPOINT;CHECKPOINT;"
-  #norm user 
-  #
-  #
+
+backupEnv: |
+  #!/bin/bash
+  export USE_WALG={{ .Values.backup.enabled | quote }}
+  export BACKUP_NUM_TO_RETAIN={{ .Values.backup.retainBackups | quote}}
+  export WALG_BACKUP_THRESHOLD_MEGABYTES={{ .Values.backup.backupThresholdMegabytes | quote }}
+  export WALE_BACKUP_THRESHOLD_PERCENTAGE={{ .Values.backup.backupThresholdPercentage | quote }}
+  export AWS_ACCESS_KEY_ID={{ .Values.backup.s3.awsAccessKeyID | quote }}
+  export AWS_SECRET_ACCESS_KEY={{ .Values.backup.s3.awsSecretAccessKey | quote }}
+  export WALG_S3_PREFIX={{ .Values.backup.s3.walGS3Prefix | quote }}
+  export AWS_ENDPOINT={{ .Values.backup.s3.awsEndpoint | quote }}
+  export AWS_S3_FORCE_PATH_STYLE={{ .Values.backup.s3.awsS3ForcePathStyle | quote }}
+  export AWS_REGION={{ .Values.backup.s3.awsRegion | quote }}
+
 postgresql: 
   config: |-
     log_min_duration_statement = 1000
@@ -269,7 +276,6 @@ backup:
     awsS3ForcePathStyle: "true"
     awsRegion: dx-1
 
-
 persistentVolume:
   enabled: true
   size: 10G