Skip to content

Commit 9b94b44

Browse files
zhangeamonzhangeamon
zhangeamon
authored and
zhangeamon
committed
chore(postgresql): add logical backup
1 parent f04b3cf commit 9b94b44

9 files changed

Lines changed: 211 additions & 21 deletions

File tree

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,4 @@
11
contrib/ci/tmp
2-
.vscode/
2+
.vscode/
3+
charts/database/Chart.lock
4+
charts/database/charts/

‎Dockerfile‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,31 +10,34 @@ ARG PYTHON_VERSION="3.13" \
1010

1111
ENV HOME=/data \
1212
PG_MAJOR=18 \
13-
PG_MINOR=1
14-
ENV PGDATA $HOME/$PG_MAJOR
13+
PG_MINOR=1 \
14+
S3_RCLONE_VERSION="1.71.1"
1515

16-
RUN install-packages vim gcc \
16+
ENV PGDATA=$HOME/$PG_MAJOR
17+
18+
RUN install-packages vim gcc pigz jq\
1719
&& install-stack python $PYTHON_VERSION \
1820
&& install-stack postgresql $PG_MAJOR.$PG_MINOR \
21+
&& install-stack rclone $S3_RCLONE_VERSION \
1922
&& install-stack postgres_exporter $POSTGRES_EXPORTER_VERSION \
2023
&& . init-stack \
2124
&& set -eux; pip3 install --disable-pip-version-check --no-cache-dir psycopg[binary] patroni[kubernetes] 2>/dev/null; set +eux \
2225
&& apt-get purge -y --auto-remove gcc \
2326
&& apt-get autoremove -y \
2427
&& apt-get clean -y \
2528
&& rm -rf \
26-
/usr/share/doc \
27-
/usr/share/man \
28-
/usr/share/info \
29-
/usr/share/locale \
30-
/var/lib/apt/lists/* \
31-
/var/log/* \
32-
/var/cache/debconf/* \
33-
/etc/systemd \
34-
/lib/lsb \
35-
/lib/udev \
36-
/usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \
37-
/usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \
29+
/usr/share/doc \
30+
/usr/share/man \
31+
/usr/share/info \
32+
/usr/share/locale \
33+
/var/lib/apt/lists/* \
34+
/var/log/* \
35+
/var/cache/debconf/* \
36+
/etc/systemd \
37+
/lib/lsb \
38+
/lib/udev \
39+
/usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \
40+
/usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \
3841
&& mkdir -p /usr/share/man/man{1..8} \
3942
&& mkdir -p $PGDATA \
4043
&& groupadd postgres && useradd -g postgres postgres \

‎charts/database/templates/_helper.tpl‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,51 @@
11
{{- define "database.envs" }}
22
env:
3+
- name: RETAIN_BACKUPS_AGE
4+
value: "{{.Values.cronjob.retainBackupsAge}}"
5+
{{- if (.Values.storageEndpoint) }}
6+
- name: "DRYCC_STORAGE_BUCKET"
7+
valueFrom:
8+
secretKeyRef:
9+
name: database-creds
10+
key: storage-bucket
11+
- name: "DRYCC_STORAGE_ENDPOINT"
12+
valueFrom:
13+
secretKeyRef:
14+
name: database-creds
15+
key: storage-endpoint
16+
- name: "DRYCC_STORAGE_ACCESSKEY"
17+
valueFrom:
18+
secretKeyRef:
19+
name: database-creds
20+
key: storage-accesskey
21+
- name: "DRYCC_STORAGE_SECRETKEY"
22+
valueFrom:
23+
secretKeyRef:
24+
name: database-creds
25+
key: storage-secretkey
26+
- name: "DRYCC_STORAGE_PATH_STYLE"
27+
valueFrom:
28+
secretKeyRef:
29+
name: database-creds
30+
key: storage-path-style
31+
{{- else if .Values.storage.enabled }}
32+
- name: "DRYCC_STORAGE_BUCKET"
33+
value: "database"
34+
- name: "DRYCC_STORAGE_ENDPOINT"
35+
value: http://drycc-storage:9000
36+
- name: "DRYCC_STORAGE_ACCESSKEY"
37+
valueFrom:
38+
secretKeyRef:
39+
name: storage-creds
40+
key: accesskey
41+
- name: "DRYCC_STORAGE_SECRETKEY"
42+
valueFrom:
43+
secretKeyRef:
44+
name: storage-creds
45+
key: secretkey
46+
- name: "DRYCC_STORAGE_PATH_STYLE"
47+
value: "true"
48+
{{- end }}
349
{{- if eq .Values.debug "true" }}
450
- name: PATRONI_LOG_LEVEL
551
value: DEBUG

‎charts/database/templates/database-cronjob-backup.yaml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{{- if .Values.cronjob.enabled }}
2+
apiVersion: {{ include "common.capabilities.cronjob.apiVersion" . }}
3+
kind: CronJob
4+
metadata:
5+
name: drycc-database-cronjob-backup
6+
labels:
7+
heritage: drycc
8+
annotations:
9+
component.drycc.cc/version: {{ .Values.imageTag }}
10+
11+
spec:
12+
schedule: "{{ .Values.cronjob.scheduleCronJob }}"
13+
failedJobsHistoryLimit: 1
14+
successfulJobsHistoryLimit: 1
15+
jobTemplate:
16+
spec:
17+
template:
18+
spec:
19+
restartPolicy: OnFailure
20+
containers:
21+
- name: drycc-database-cronjob-backup
22+
image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/database:{{.Values.imageTag}}
23+
imagePullPolicy: {{.Values.imagePullPolicy}}
24+
command:
25+
- init-stack
26+
args:
27+
- /usr/share/scripts/backup.sh
28+
{{- include "database.envs" . | indent 12 }}
29+
{{- end -}}

‎charts/database/templates/database-secret-creds.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,10 @@ data:
1212
replicator-password: {{ include "common.secrets.lookup" (dict "secret" "database-creds" "key" "replicator-password" "defaultValue" (.Values.replicatorPassword | default (randAlphaNum 32) | lower) "context" $) }}
1313
user: {{ include "common.secrets.lookup" (dict "secret" "database-creds" "key" "user" "defaultValue" (.Values.user | default (randAlpha 32) | lower) "context" $) }}
1414
password: {{ include "common.secrets.lookup" (dict "secret" "database-creds" "key" "password" "defaultValue" (.Values.password | default (randAlphaNum 32)) "context" $) }}
15+
{{- if (.Values.storageEndpoint) }}
16+
storage-bucket: {{ .Values.storageBucket | b64enc }}
17+
storage-endpoint: {{ .Values.storageEndpoint | b64enc }}
18+
storage-accesskey: {{ .Values.storageAccesskey | b64enc }}
19+
storage-secretkey: {{ .Values.storageSecretkey | b64enc }}
20+
storage-path-style: {{ .Values.storagePathStyle | b64enc }}
21+
{{- end }}

‎charts/database/values.yaml‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,24 @@ podAntiAffinityPreset:
4545
app: "drycc-database"
4646

4747
debug: "false"
48+
# Patroni pg_ctl timeout
4849
timeout: "1200"
50+
51+
# The following parameters will no longer use the built-in storage component.
52+
storageBucket: "database"
53+
storageEndpoint: ""
54+
storageAccesskey: ""
55+
storageSecretkey: ""
56+
storagePathStyle: "true"
57+
58+
storage:
59+
enabled: true
60+
61+
cronjob:
62+
enabled: true
63+
scheduleCronJob: "0 3 * * *"
64+
retainBackupsAge: 30d
65+
4966
# Service
5067
service:
5168
# Provide any additional service annotations
@@ -66,7 +83,7 @@ persistence:
6683
storageClass: ""
6784
# The username and password to be used by the on-cluster database.
6885
# If left empty they will be generated
69-
# The user name should be set to lowercase letters
86+
# The user name should be set to lowercase letters
7087
superuser: "postgres"
7188
superuserPassword: ""
7289
replicator: "standby"

‎rootfs/entrypoint.sh‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
#!/bin/bash
2-
32
if [[ $UID -ge 10000 ]]; then
43
GID=$(id -g)
54
sed -e "s/^postgres:x:[^:]*:[^:]*:/postgres:x:$UID:$GID:/" /etc/passwd > /tmp/passwd
@@ -19,14 +18,14 @@ bootstrap:
1918
use_pg_rewind: true
2019
use_slots: true
2120
initdb:
22-
- auth-host: md5
21+
- auth-host: scram-sha-256
2322
- auth-local: trust
2423
- encoding: UTF8
2524
- locale: ${LANG}
2625
- data-checksums
2726
pg_hba:
28-
- host all all 0.0.0.0/0 md5
29-
- host replication ${DRYCC_DATABASE_REPLICATOR} ${PATRONI_KUBERNETES_POD_IP}/16 md5
27+
- host all all 0.0.0.0/0 scram-sha-256
28+
- host replication ${DRYCC_DATABASE_REPLICATOR} ${PATRONI_KUBERNETES_POD_IP}/16 scram-sha-256
3029
post_bootstrap: /usr/share/scripts/patroni/post_init.sh
3130
restapi:
3231
connect_address: '${PATRONI_KUBERNETES_POD_IP}:8008'
@@ -43,13 +42,15 @@ postgresql:
4342
max_prepared_transactions: 0
4443
max_locks_per_transaction: 64
4544
wal_log_hints: "on"
45+
wal_level: logical
4646
track_commit_timestamp: "off"
4747
archive_mode: "on"
4848
archive_timeout: 300s
4949
archive_command: "/bin/true"
5050
log_min_duration_statement: 1000
5151
log_lock_waits: on
5252
log_statement: 'ddl'
53+
jit: off
5354
connect_address: '${PATRONI_KUBERNETES_POD_IP}:5432'
5455
authentication:
5556
superuser:

‎rootfs/usr/share/scripts/backup.sh‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
#!/bin/bash
2+
set -e
3+
4+
# Setup Rclone
5+
/usr/share/scripts/create_bucket
6+
7+
# Backup PostgreSQL databases to S3-compatible object storage
8+
export PGPASSWORD=$DRYCC_DATABASE_SUPERUSER_PASSWORD
9+
PGPORT=$DRYCC_DATABASE_REPLICA_SERVICE_PORT_POSTGRES
10+
PGUSER=postgres
11+
POSTGRES_HOST=$DRYCC_DATABASE_REPLICA_SERVICE_HOST
12+
RETAIN_BACKUPS_AGE=$RETAIN_BACKUPS_AGE
13+
14+
# PostgreSQL global objects backup
15+
BACKUP_PATH="${DRYCC_STORAGE_BUCKET}/$(date +%Y%m%d%H%M)"
16+
echo "DB: ${POSTGRES_USER}@${POSTGRES_HOST}"
17+
echo "S3 Storage: storage:${BACKUP_PATH}"
18+
# Backup global objects
19+
if pg_dumpall -g -U "${POSTGRES_USER}" -h "${POSTGRES_HOST}" \
20+
| pigz -c -p 4 -6 \
21+
| rclone rcat \
22+
"storage:${BACKUP_PATH}/roles_globals.sql.gz" \
23+
--bwlimit 10M \
24+
--transfers 4 \
25+
--s3-chunk-size 64M \
26+
--stats 5s \
27+
--progress \
28+
--retries 3; then
29+
echo "✅ PostgreSQL global objects backup complated!"
30+
fi
31+
# fetch the list of databases
32+
DATABASES=$(psql -U "$POSTGRES_USER" -h "$POSTGRES_HOST" -t -c "SELECT datname FROM pg_database WHERE datistemplate = false;")
33+
34+
# backup each database individually
35+
for DB in $DATABASES; do
36+
echo "Backing up $DB to $MINIO_PATH/$DB.sql.gz"
37+
if pg_dump -U "${POSTGRES_USER}" -h "${POSTGRES_HOST}" "$DB" \
38+
| pigz -c -p 4 -6 \
39+
| rclone rcat \
40+
"storage:${BACKUP_PATH}/$DB.sql.gz" \
41+
--bwlimit 10M \
42+
--transfers 4 \
43+
--s3-chunk-size 64M \
44+
--stats 5s \
45+
--progress \
46+
--retries 3; then
47+
echo "✅ PostgreSQL $DB global objects backup completed!"
48+
fi
49+
done
50+
51+
echo "Backup process completed!"
52+
53+
echo "delete storage before ${RETAIN_BACKUPS_AGE} ..."
54+
rclone delete "storage:${DRYCC_STORAGE_BUCKET}" \
55+
--min-age ${RETAIN_BACKUPS_AGE} \
56+
--include "*.sql.gz" \
57+
--dry-run \
58+
-v || true
59+
60+
rclone delete "storage:${DRYCC_STORAGE_BUCKET}" \
61+
--min-age ${RETAIN_BACKUPS_AGE} \
62+
--include "*.sql.gz" \
63+
|| true
64+
echo "delete completed."
65+
66+
echo "=== backup completed: $(date) ==="

‎rootfs/usr/share/scripts/create_bucket‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
mkdir -p ~/.config/rclone
6+
touch ~/.config/rclone/rclone.conf
7+
rclone config create storage s3 \
8+
provider=Other \
9+
access_key_id="${DRYCC_STORAGE_ACCESSKEY}" \
10+
secret_access_key="${DRYCC_STORAGE_SECRETKEY}" \
11+
endpoint="${DRYCC_STORAGE_ENDPOINT}" \
12+
force_path_style="${DRYCC_STORAGE_PATH_STYLE:-true}" --no-output
13+
14+
if ! rclone lsd storage: > /dev/null 2>&1; then
15+
sleep 9s
16+
echo "waiting for object storage to become ready..."
17+
fi
18+
19+
rclone mkdir "storage:${DRYCC_STORAGE_BUCKET}"

0 commit comments

Comments
 (0)