From 91c8331bfa966480082eeb4f9478c6259abbb430 Mon Sep 17 00:00:00 2001
From: jianxiaoguo <842666271@qq.com>
Date: Fri, 12 Aug 2022 13:51:37 +0800
Subject: [PATCH] feat(postgres): use patroni

---
 Dockerfile                                    |  56 ++++------
 Makefile                                      |   9 +-
 charts/database/templates/_helper.tpl         |  77 +++++++++----
 .../templates/database-clusterrole.yaml       |  15 +++
 .../database-clusterrolebinding.yaml          |  14 +++
 charts/database/templates/database-pvc.yaml   |  24 ----
 .../templates/database-role-binding.yaml      |  13 +++
 charts/database/templates/database-role.yaml  |  56 ++++++++++
 .../templates/database-secret-creds.yaml      |   6 +-
 .../templates/database-service-replica.yaml   |  23 ++++
 .../database/templates/database-service.yaml  |   4 +-
 ...loyment.yaml => database-statefulset.yaml} |  71 +++++++++---
 charts/database/values.yaml                   |  29 ++++-
 contrib/ci/test-storage.sh                    |  78 -------------
 contrib/ci/test.sh                            |  43 -------
 rootfs/bin/backup                             |   8 --
 rootfs/bin/create_bucket                      |  23 ----
 rootfs/bin/do_backup                          |  15 ---
 rootfs/bin/is_running                         |  11 --
 .../001_setup_envdir.sh                       |  12 --
 .../002_create_bucket.sh                      |   4 -
 .../003_restore_from_backup.sh                |  71 ------------
 .../004_run_backups.sh                        |   4 -
 rootfs/docker-entrypoint.sh                   | 105 ------------------
 rootfs/entrypoint.sh                          |  43 +++++++
 rootfs/usr/share/scripts/patroni/post_init.sh |  15 +++
 26 files changed, 341 insertions(+), 488 deletions(-)
 create mode 100644 charts/database/templates/database-clusterrole.yaml
 create mode 100644 charts/database/templates/database-clusterrolebinding.yaml
 delete mode 100644 charts/database/templates/database-pvc.yaml
 create mode 100644 charts/database/templates/database-role-binding.yaml
 create mode 100644 charts/database/templates/database-role.yaml
 create mode 100644 charts/database/templates/database-service-replica.yaml
 rename charts/database/templates/{database-deployment.yaml => database-statefulset.yaml} (54%)
 delete mode 100755 contrib/ci/test-storage.sh
 delete mode 100755 contrib/ci/test.sh
 delete mode 100755 rootfs/bin/backup
 delete mode 100755 rootfs/bin/create_bucket
 delete mode 100755 rootfs/bin/do_backup
 delete mode 100755 rootfs/bin/is_running
 delete mode 100755 rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh
 delete mode 100755 rootfs/docker-entrypoint-initdb.d/002_create_bucket.sh
 delete mode 100755 rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh
 delete mode 100755 rootfs/docker-entrypoint-initdb.d/004_run_backups.sh
 delete mode 100755 rootfs/docker-entrypoint.sh
 create mode 100755 rootfs/entrypoint.sh
 create mode 100755 rootfs/usr/share/scripts/patroni/post_init.sh

diff --git a/Dockerfile b/Dockerfile
index e69ebba..3f87a96 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,46 +1,34 @@
 FROM registry.drycc.cc/drycc/base:bullseye
 
-COPY rootfs/bin /bin/
-COPY rootfs/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d/
-COPY rootfs/docker-entrypoint.sh /docker-entrypoint.sh
-ENV JQ_VERSION="1.6" \
-  GOSU_VERSION="1.14" \
-  MC_VERSION="2022.04.01.23.44.48" \
-  WAL_G_VERSION="1.1" \
-  PYTHON_VERSION="3.10.4" \
+COPY rootfs/usr /usr/
+COPY rootfs/entrypoint.sh /entrypoint.sh
+ENV PYTHON_VERSION="3.10.4" \
   PG_MAJOR=14 \
   POSTGRESQL_VERSION="14.2"
 
-
 ENV PGDATA /data/${PG_MAJOR}
-ENV WALG_ENVDIR /etc/wal-g.d/env
 
-RUN mkdir -p $WALG_ENVDIR \
-  && install-stack jq $JQ_VERSION \
-  && install-stack gosu $GOSU_VERSION \
-  && install-stack mc $MC_VERSION \
-  && install-stack wal-g $WAL_G_VERSION \
-  && install-stack python $PYTHON_VERSION \
+RUN install-stack python $PYTHON_VERSION \
   && install-stack postgresql $POSTGRESQL_VERSION && . init-stack \
   && rm -rf \
-      /usr/share/doc \
-      /usr/share/man \
-      /usr/share/info \
-      /usr/share/locale \
-      /var/lib/apt/lists/* \
-      /var/log/* \
-      /var/cache/debconf/* \
-      /etc/systemd \
-      /lib/lsb \
-      /lib/udev \
-      /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \
-      /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \
+    /usr/share/doc \
+    /usr/share/man \
+    /usr/share/info \
+    /usr/share/locale \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /var/cache/debconf/* \
+    /etc/systemd \
+    /lib/lsb \
+    /lib/udev \
+    /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/IBM* \
+    /usr/lib/`echo $(uname -m)`-linux-gnu/gconv/EBC* \
   && mkdir -p /usr/share/man/man{1..8} \
-  && mkdir -p /run/postgresql $PGDATA \
+  && mkdir -p $PGDATA \
   && groupadd postgres && useradd -g postgres postgres \
-  && chown -R postgres:postgres /run/postgresql $PGDATA \
-  && set -eux; pip3 install --disable-pip-version-check --no-cache-dir envdir 2>/dev/null
+  && chown -R postgres:postgres /data \
+  && set -eux; pip3 install --disable-pip-version-check --no-cache-dir psycopg[binary] patroni[kubernetes] 2>/dev/null
 
-ENTRYPOINT ["init-stack", "/docker-entrypoint.sh"]
-CMD ["postgres"]
-EXPOSE 5432
+USER postgres
+ENTRYPOINT ["init-stack", "/entrypoint.sh"]
+EXPOSE 5432 8008
diff --git a/Makefile b/Makefile
index d08bf15..5eaa3de 100644
--- a/Makefile
+++ b/Makefile
@@ -9,7 +9,7 @@ PLATFORM ?= linux/amd64,linux/arm64
 
 include versioning.mk
 
-SHELL_SCRIPTS = $(wildcard _scripts/*.sh contrib/ci/*.sh rootfs/bin/*)
+SHELL_SCRIPTS = $(wildcard rootfs/usr/share/scripts/patroni/*)
 
 # The following variables describe the containerized development environment
 # and other build options
@@ -29,14 +29,9 @@ docker-build:
 docker-buildx:
 	docker buildx build --platform ${PLATFORM} -t ${IMAGE} . --push
 
-test: test-style test-functional
+test: test-style
 
 test-style:
 	${DEV_ENV_CMD} shellcheck $(SHELL_SCRIPTS)
 
-test-functional: test-functional-storage
-
-test-functional-storage:
-	contrib/ci/test-storage.sh ${IMAGE}
-
 .PHONY: all docker-build docker-push test
diff --git a/charts/database/templates/_helper.tpl b/charts/database/templates/_helper.tpl
index 1aeeba9..0cf961a 100644
--- a/charts/database/templates/_helper.tpl
+++ b/charts/database/templates/_helper.tpl
@@ -2,48 +2,77 @@
 env:
 - name: DATABASE_STORAGE
   value: "{{.Values.global.storage}}"
+{{- if eq .Values.debug "true" }}
+- name: PATRONI_LOG_LEVEL
+  value: DEBUG
+- name: PATRONI_LOG_TRACEBACK_LEVEL
+  value: DEBUG
+{{- end }}
 - name: PGCTLTIMEOUT
   value: "{{.Values.timeout}}"
-- name: "DRYCC_DATABASE_USER"
+- name: PATRONI_KUBERNETES_POD_IP
   valueFrom:
-    secretKeyRef:
-      name: database-creds
-      key: user
-- name: "DRYCC_DATABASE_PASSWORD"
+    fieldRef:
+      fieldPath: status.podIP
+- name: PATRONI_KUBERNETES_NAMESPACE
+  valueFrom:
+    fieldRef:
+      fieldPath: metadata.namespace
+- name: PATRONI_KUBERNETES_BYPASS_API_SERVICE
+  value: 'true'
+- name: PATRONI_KUBERNETES_USE_ENDPOINTS
+  value: 'true'
+- name: PATRONI_KUBERNETES_LABELS
+  value: '{app: drycc-database, cluster-name: drycc-database}'
+- name: PATRONI_SCOPE
+  value: drycc-database
+- name: PATRONI_NAME
+  valueFrom:
+    fieldRef:
+      fieldPath: metadata.name
+- name: PATRONI_POSTGRESQL_PGPASS
+  value: /tmp/pgpass
+- name: PATRONI_POSTGRESQL_LISTEN
+  value: '0.0.0.0:5432'
+- name: PATRONI_RESTAPI_LISTEN
+  value: '0.0.0.0:8008'
+- name: DRYCC_DATABASE_SUPERUSER
   valueFrom:
     secretKeyRef:
       name: database-creds
-      key: password
-- name: "DRYCC_STORAGE_LOOKUP"
+      key: superuser
+- name: DRYCC_DATABASE_SUPERUSER_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: storage-creds
-      key: lookup
-- name: "DRYCC_STORAGE_HEALTH"
+      name: database-creds
+      key: superuser-password
+- name: DRYCC_DATABASE_REPLICATOR
   valueFrom:
     secretKeyRef:
-      name: storage-creds
-      key: health
-- name: "DRYCC_STORAGE_BUCKET"
+      name: database-creds
+      key: replicator
+- name: DRYCC_DATABASE_REPLICATOR_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: storage-creds
-      key: database-bucket
-- name: "DRYCC_STORAGE_ENDPOINT"
+      name: database-creds
+      key: replicator-password
+- name: "DRYCC_DATABASE_USER"
   valueFrom:
     secretKeyRef:
-      name: storage-creds
-      key: endpoint
-- name: "DRYCC_STORAGE_ACCESSKEY"
+      name: database-creds
+      key: user
+- name: "DRYCC_DATABASE_PASSWORD"
   valueFrom:
     secretKeyRef:
-      name: storage-creds
-      key: accesskey
-- name: "DRYCC_STORAGE_SECRETKEY"
+      name: database-creds
+      key: password
+{{- if eq .Values.global.storageLocation "on-cluster" }}
+- name: "DRYCC_STORAGE_HEALTH"
   valueFrom:
     secretKeyRef:
       name: storage-creds
-      key: secretkey
+      key: health
+{{- end }}
 {{- end }}
 
 {{/* Generate database deployment limits */}}
@@ -64,4 +93,4 @@ resources:
     hugepages-1Gi: {{.Values.limitsHugepages1Gi}}
     {{- end }}
 {{- end }}
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/charts/database/templates/database-clusterrole.yaml b/charts/database/templates/database-clusterrole.yaml
new file mode 100644
index 0000000..11b2f70
--- /dev/null
+++ b/charts/database/templates/database-clusterrole.yaml
@@ -0,0 +1,15 @@
+{{- if eq .Values.global.databaseLocation "on-cluster" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: drycc-database
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  resourceNames:
+  - kubernetes
+  verbs:
+  - get
+{{- end }}
diff --git a/charts/database/templates/database-clusterrolebinding.yaml b/charts/database/templates/database-clusterrolebinding.yaml
new file mode 100644
index 0000000..11877bc
--- /dev/null
+++ b/charts/database/templates/database-clusterrolebinding.yaml
@@ -0,0 +1,14 @@
+{{- if eq .Values.global.databaseLocation "on-cluster" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: drycc-database
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: drycc-database
+subjects:
+- kind: ServiceAccount
+  name: drycc-database
+  namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/database/templates/database-pvc.yaml b/charts/database/templates/database-pvc.yaml
deleted file mode 100644
index 1de3839..0000000
--- a/charts/database/templates/database-pvc.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-{{- if .Values.persistence.enabled }}
-kind: PersistentVolumeClaim
-apiVersion: v1
-metadata:
-  name: drycc-database
-  labels:
-    heritage: drycc
-spec:
-  accessModes:
-    - {{ .Values.persistence.accessMode | quote }}
-  resources:
-    requests:
-      storage: {{ .Values.persistence.size | quote }}
-{{- if .Values.persistence.storageClass }}
-{{- if (eq "-" .Values.persistence.storageClass) }}
-  storageClassName: ""
-{{- else }}
-  storageClassName: "{{ .Values.persistence.storageClass }}"
-{{- end }}
-{{- end }}
-{{- if .Values.persistence.volumeName }}
-  volumeName: "{{ .Values.persistence.volumeName }}"
-{{- end }}
-{{- end }}
\ No newline at end of file
diff --git a/charts/database/templates/database-role-binding.yaml b/charts/database/templates/database-role-binding.yaml
new file mode 100644
index 0000000..bbca50b
--- /dev/null
+++ b/charts/database/templates/database-role-binding.yaml
@@ -0,0 +1,13 @@
+{{- if eq .Values.global.databaseLocation "on-cluster" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: drycc-database
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: drycc-database
+subjects:
+- kind: ServiceAccount
+  name: drycc-database
+{{- end }}
diff --git a/charts/database/templates/database-role.yaml b/charts/database/templates/database-role.yaml
new file mode 100644
index 0000000..5ed547c
--- /dev/null
+++ b/charts/database/templates/database-role.yaml
@@ -0,0 +1,56 @@
+{{- if eq .Values.global.databaseLocation "on-cluster" }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: drycc-database
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - update
+  - watch
+  # delete and deletecollection are required only for 'patronictl remove'
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - patch
+  - update
+  # the following three privileges are necessary only when using endpoints
+  - create
+  - list
+  - watch
+  # delete and deletecollection are required only for for 'patronictl remove'
+  - delete
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - patch
+  - update
+  - watch
+# The following privilege is only necessary for creation of headless service
+# for drycc-database-config endpoint, in order to prevent cleaning it up by the
+# k8s master. You can avoid giving this privilege by explicitly creating the
+# service like it is done in this manifest (lines 2..10)
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+{{- end }}
diff --git a/charts/database/templates/database-secret-creds.yaml b/charts/database/templates/database-secret-creds.yaml
index dba1870..f689f15 100644
--- a/charts/database/templates/database-secret-creds.yaml
+++ b/charts/database/templates/database-secret-creds.yaml
@@ -9,6 +9,10 @@ metadata:
   annotations:
     "helm.sh/hook": pre-install
 data:
-  user: {{ if .Values.user | default "" | ne "" }}{{ .Values.user | b64enc }}{{ else }}{{ randAlphaNum 32 | b64enc }}{{ end }}
+  superuser: {{ if .Values.superuser | default "" | ne "" }}{{ .Values.superuser | b64enc }}{{ else }}{{ randAlpha 32 | b64enc }}{{ end }}
+  superuser-password: {{ if .Values.superuserPassword | default "" | ne "" }}{{ .Values.superuserPassword | b64enc }}{{ else }}{{ randAlphaNum 32 | b64enc }}{{ end }}
+  replicator: {{ if .Values.replicator | default "" | ne "" }}{{ .Values.replicator | b64enc }}{{ else }}{{ randAlpha 32 | b64enc }}{{ end }}
+  replicator-password: {{ if .Values.replicatorPassword | default "" | ne "" }}{{ .Values.replicatorPassword | b64enc }}{{ else }}{{ randAlphaNum 32 | b64enc }}{{ end }}
+  user: {{ if .Values.user | default "" | ne "" }}{{ .Values.user | b64enc }}{{ else }}{{ randAlpha 32 | b64enc }}{{ end }}
   password: {{ if .Values.password | default "" | ne "" }}{{ .Values.password | b64enc }}{{ else }}{{ randAlphaNum 32 | b64enc }}{{ end }}
 {{- end }}
diff --git a/charts/database/templates/database-service-replica.yaml b/charts/database/templates/database-service-replica.yaml
new file mode 100644
index 0000000..0bc8c00
--- /dev/null
+++ b/charts/database/templates/database-service-replica.yaml
@@ -0,0 +1,23 @@
+{{- if eq .Values.global.databaseLocation "on-cluster" }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: drycc-database-replica
+  annotations:
+    {{- with .Values.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    heritage: drycc
+    app: drycc-database
+    role: replica
+spec:
+  type: ClusterIP
+  ports:
+    - name: postgres
+      port: 5432
+      targetPort: 5432
+  selector:
+    app: drycc-database
+    role: replica
+{{- end }}
diff --git a/charts/database/templates/database-service.yaml b/charts/database/templates/database-service.yaml
index cc95aa7..2648b5c 100644
--- a/charts/database/templates/database-service.yaml
+++ b/charts/database/templates/database-service.yaml
@@ -10,9 +10,9 @@ metadata:
   labels:
     heritage: drycc
 spec:
+  type: ClusterIP
   ports:
     - name: postgres
       port: 5432
-  selector:
-    app: drycc-database
+      targetPort: 5432
 {{- end }}
diff --git a/charts/database/templates/database-deployment.yaml b/charts/database/templates/database-statefulset.yaml
similarity index 54%
rename from charts/database/templates/database-deployment.yaml
rename to charts/database/templates/database-statefulset.yaml
index 313f4b2..61280d0 100644
--- a/charts/database/templates/database-deployment.yaml
+++ b/charts/database/templates/database-statefulset.yaml
@@ -1,6 +1,6 @@
 {{- if eq .Values.global.databaseLocation "on-cluster" }}
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   name: drycc-database
   labels:
@@ -8,22 +8,24 @@ metadata:
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
 spec:
-  replicas: 1
-  strategy:
-    type: Recreate
+  replicas: {{ .Values.replicas }}
+  serviceName: drycc-database
   selector:
     matchLabels:
       app: drycc-database
+      cluster-name: drycc-database
   template:
     metadata:
       labels:
         app: drycc-database
+        cluster-name: drycc-database
     spec:
       affinity:
         podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset.type "key" .Values.podAffinityPreset.key "values" .Values.podAffinityPreset.values ) | nindent 10 }}
         podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset.type "key" .Values.podAntiAffinityPreset.key "values" .Values.podAntiAffinityPreset.values ) | nindent 10 }}
         nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values ) | nindent 10 }}
-      serviceAccount: drycc-database
+      serviceAccountName: drycc-database
+      {{- if eq .Values.global.storageLocation "on-cluster" }}
       initContainers:
       - name: drycc-database-init
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
@@ -34,36 +36,71 @@ spec:
           - -g
           - $(DRYCC_STORAGE_HEALTH)
         {{- include "database.envs" . | indent 8 }}
+      {{- end }}
       containers:
       - name: drycc-database
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/postgres:{{.Values.imageTag}}
         imagePullPolicy: {{.Values.imagePullPolicy}}
         ports:
+        - containerPort: 8008
+          protocol: TCP
         - containerPort: 5432
+          protocol: TCP
         {{- include "database.limits" . | indent 8 }}
         {{- include "database.envs" . | indent 8 }}
         lifecycle:
           preStop:
             exec:
               command:
-              - init-stack
-              - gosu
-              - postgres
-              - do_backup
+                - /usr/bin/env
+                - bash
+                - -c
+                - |
+                  # switch leader pod if the current pod is the leader
+                  if curl --fail http://localhost:8008/read-write; then
+                    init-stack patronictl switchover --force
+                  fi
+        livenessProbe:
+          failureThreshold: 3                            
+          httpGet:       
+            path: /liveness
+            port: 8008
+            scheme: HTTP   
+          initialDelaySeconds: 10
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 5
         readinessProbe:
           exec:
             command:
-            - init-stack
-            - is_running
-          initialDelaySeconds: 30
-          timeoutSeconds: 1
-      {{- if .Values.persistence.enabled }}
+              - init-stack
+              - pg_isready
+              - -q
+          initialDelaySeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
         volumeMounts:
         - name: database-data
           mountPath: /data
+  {{- if .Values.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: database-data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      {{- if .Values.persistence.storageClass }}
+      {{- if (eq "-" .Values.persistence.storageClass) }}
+      storageClassName: ""
+      {{- else }}
+      storageClassName: "{{ .Values.persistence.storageClass }}"
+      {{- end }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.persistence.size | quote }}
+  {{- else }}
       volumes:
       - name: database-data
-        persistentVolumeClaim:
-          claimName: drycc-database
-      {{- end }}
+        emptyDir: {}
+  {{- end }}
 {{- end }}
diff --git a/charts/database/values.yaml b/charts/database/values.yaml
index 9edcc4d..97a62bb 100644
--- a/charts/database/values.yaml
+++ b/charts/database/values.yaml
@@ -7,6 +7,8 @@ imageRegistry: "registry.drycc.cc"
 # limitsHugepages2Mi: 100Mi
 # limitsHugepages1Gi: 2Gi
 
+replicas: 3
+
 nodeAffinityPreset:
   key: "drycc.cc/node"
   type: "soft"
@@ -25,6 +27,7 @@ podAntiAffinityPreset:
   values:
   - "drycc-database"
 
+debug: "false"
 timeout: "1200"
 # Service
 service:
@@ -33,7 +36,6 @@ service:
 # GCP PDs and EBS volumes are supported only
 persistence:
   enabled: false # Set to true to enable persistence
-  accessMode: ReadWriteOnce
   size: 5Gi # PVC size
   ## database data Persistent Volume Storage Class
   ## If defined, storageClassName: <storageClass>
@@ -45,10 +47,27 @@ persistence:
   ## Storage class of PV to bind. By default it looks for standard storage class.
   ## If the PV uses a different storage class, specify that here.
   storageClass: ""
-  volumeName: ""
 # The username and password to be used by the on-cluster database.
 # If left empty they will be generated using randAlphaNum
-username: ""
+superuser: "postgres"
+superuserPassword: ""
+replicator: "standby"
+replicatorPassword: ""
+user: ""
 password: ""
-passportDatabaseName: "drycc_passport"
-controllerDatabaseName: "drycc_controller"
+
+global:
+  # Set the location of Workflow's Object Storage
+  #
+  # Valid values are:
+  # - on-cluster: Run drycc storage within the Kubernetes cluster
+  # - off-cluster: Run Storage outside the Kubernetes cluster (Compatible with s3 API)
+  storageLocation: "on-cluster"
+  # Set the location of Workflow's PostgreSQL database
+  #
+  # Valid values are:
+  # - on-cluster: Run PostgreSQL within the Kubernetes cluster (credentials are generated
+  #   automatically; backups are sent to object storage
+  #   configured above)
+  # - off-cluster: Run PostgreSQL outside the Kubernetes cluster
+  databaseLocation: "on-cluster"
diff --git a/contrib/ci/test-storage.sh b/contrib/ci/test-storage.sh
deleted file mode 100755
index 5b8e63d..0000000
--- a/contrib/ci/test-storage.sh
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/usr/bin/env bash
-
-set -eof pipefail
-
-cleanup() {
-  kill-containers "${STORAGE_JOB}" "${PG_JOB}"
-}
-trap cleanup EXIT
-
-TEST_ROOT=$(dirname "${BASH_SOURCE[0]}")/
-# shellcheck source=/dev/null
-source "${TEST_ROOT}/test.sh"
-
-# make sure we are in this dir
-CURRENT_DIR=$(cd "$(dirname "$0")"; pwd)
-
-create-postgres-creds
-
-puts-step "creating fake storage credentials"
-
-s3Accesskey="1234567890123456789012345678901234567890"
-s3Secretkey="1234567890123456789012345678901234567890"
-# boot storage
-mkdir -p "${CURRENT_DIR}"/tmp/bin
-echo "ls /data/database/*/basebackups_005" > "${CURRENT_DIR}"/tmp/bin/backups.sh
-STORAGE_JOB=$(docker run -d \
-  -e DRYCC_STORAGE_ACCESSKEY=$s3Accesskey \
-  -e DRYCC_STORAGE_SECRETKEY=$s3Secretkey \
-  -v "${CURRENT_DIR}"/tmp/bin:/tmp/bin \
-  "${DEV_REGISTRY}"/drycc/storage:canary minio server /data/)
-
-puts-step "storage starting, wait 30s."
-sleep 30
-
-# boot postgres, linking the storage container and setting DRYCC_STORAGE_ENDPOINT
-STORAGE_IP=$(docker inspect --format "{{ .NetworkSettings.IPAddress }}" "${STORAGE_JOB}")
-PG_CMD="docker run -d \
-  --add-host storage.local:${STORAGE_IP} \
-  -e PGCTLTIMEOUT=1200 \
-  -e BACKUP_FREQUENCY=1s \
-  -e DATABASE_STORAGE=storage \
-  -e DRYCC_STORAGE_LOOKUP=path \
-  -e DRYCC_STORAGE_BUCKET=database \
-  -e DRYCC_STORAGE_ENDPOINT=http://storage.local:9000 \
-  -e DRYCC_STORAGE_ACCESSKEY=$s3Accesskey \
-  -e DRYCC_STORAGE_SECRETKEY=$s3Secretkey $1"
-
-start-postgres "${PG_CMD}"
-
-# display logs for debugging purposes
-puts-step "displaying storage logs"
-docker logs "${STORAGE_JOB}"
-
-check-postgres "${PG_JOB}"
-
-# check if storage has the 5 backups
-puts-step "checking if storage has 5 backups"
-BACKUPS="$(docker exec "${STORAGE_JOB}" sh /tmp/bin/backups.sh | grep json)"
-
-NUM_BACKUPS="$(echo "${BACKUPS}" | wc -w)"
-# NOTE (bacongobbler): the BACKUP_FREQUENCY is only 1 second, so we could technically be checking
-# in the middle of a backup. Instead of failing, let's consider N+1 backups an acceptable case
-if [[ "${NUM_BACKUPS}" -lt "5" ]]; then
-  puts-error "the number of backups is less than 5 (found $NUM_BACKUPS)"
-  puts-error "${BACKUPS}"
-  exit 1
-fi
-
-# kill off postgres, then reboot and see if it's running after recovering from backups
-puts-step "shutting off postgres, then rebooting to test data recovery"
-kill-containers "${PG_JOB}"
-
-start-postgres "${PG_CMD}"
-
-check-postgres "${PG_JOB}"
-
-puts-step "tests PASSED!"
-exit 0
diff --git a/contrib/ci/test.sh b/contrib/ci/test.sh
deleted file mode 100755
index 55d2b20..0000000
--- a/contrib/ci/test.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-
-set -eof pipefail
-
-puts-step() {
-  echo "-----> $*"
-}
-
-puts-error() {
-  echo "!!!    $*"
-}
-
-kill-containers() {
-	puts-step "destroying containers $*"
-	docker rm -f "$@"
-}
-
-create-postgres-creds() {
-  puts-step "creating fake postgres credentials"
-
-  # create fake postgres credentials
-  mkdir -p "${CURRENT_DIR}"/tmp/creds
-  echo "testuser" > "${CURRENT_DIR}"/tmp/creds/user
-  echo "icanttellyou" > "${CURRENT_DIR}"/tmp/creds/password
-}
-
-start-postgres() {
-  export PG_JOB
-  PG_JOB=$($1)
-  # wait for postgres to boot
-  puts-step "sleeping for 30s while postgres is booting..."
-  sleep 30s
-}
-
-check-postgres() {
-  # display logs for debugging purposes
-  puts-step "displaying postgres logs"
-  docker logs "$1"
-
-  # check if postgres is running
-  puts-step "checking if postgres is running"
-  docker exec "$1" init-stack is_running
-}
diff --git a/rootfs/bin/backup b/rootfs/bin/backup
deleted file mode 100755
index a56f52e..0000000
--- a/rootfs/bin/backup
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-
-export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-4h}
-
-while true; do
-    sleep "$BACKUP_FREQUENCY"
-    do_backup
-done
diff --git a/rootfs/bin/create_bucket b/rootfs/bin/create_bucket
deleted file mode 100755
index 0a58efe..0000000
--- a/rootfs/bin/create_bucket
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-mc config host add storage \
-  "${DRYCC_STORAGE_ENDPOINT}" \
-  "${DRYCC_STORAGE_ACCESSKEY}" \
-  "${DRYCC_STORAGE_SECRETKEY}" \
-  --lookup "${DRYCC_STORAGE_LOOKUP}" \
-  --api s3v4
-
-has_bucket(){
-    mc ls storage -json|jq -r '.key'|grep -w "${DRYCC_STORAGE_BUCKET}"
-}
-
-if  [ -z "$(has_bucket)" ] ;then
-    mc mb storage/"${DRYCC_STORAGE_BUCKET}"
-    if  [ -z "$(has_bucket)" ] ;then
-        echo "create bucket ${DRYCC_STORAGE_BUCKET} error"
-        exit 1
-    fi
-fi
-echo "create bucket ${DRYCC_STORAGE_BUCKET} success"
diff --git a/rootfs/bin/do_backup b/rootfs/bin/do_backup
deleted file mode 100755
index dfe8766..0000000
--- a/rootfs/bin/do_backup
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
-
-echo "Performing a base backup..."
-if [[ -f "$PGDATA/recovery.signal" ]] ; then
-    echo "Database is currently recovering from a backup. Aborting"
-    sleep 9
-else
-    # perform a backup
-    envdir "$WALG_ENVDIR" wal-g backup-push "$PGDATA"
-    # only retain the latest BACKUPS_TO_RETAIN backups
-    envdir "$WALG_ENVDIR" wal-g delete --confirm retain "$BACKUPS_TO_RETAIN"
-    echo "Backup has been completed."
-fi
diff --git a/rootfs/bin/is_running b/rootfs/bin/is_running
deleted file mode 100755
index 0286977..0000000
--- a/rootfs/bin/is_running
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-# fail fast
-set -e
-
-if [[ -f "$PGDATA/recovery.signal" ]]; then
-  # postgres is in recovery mode, so we know it's not ready to accept incoming connections.
-  exit 1
-fi
-
-gosu postgres pg_ctl status
diff --git a/rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh b/rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh
deleted file mode 100755
index 8f25c47..0000000
--- a/rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-cd "$WALG_ENVDIR"
-
-echo $DRYCC_STORAGE_ACCESSKEY > AWS_ACCESS_KEY_ID
-echo $DRYCC_STORAGE_SECRETKEY > AWS_SECRET_ACCESS_KEY
-echo "s3://$DRYCC_STORAGE_BUCKET/$PG_MAJOR" > WALE_S3_PREFIX
-echo "${DRYCC_STORAGE_ENDPOINT}" > AWS_ENDPOINT
-if [ "${DRYCC_STORAGE_LOOKUP}" == "path" ]; then
-  echo "true" > AWS_S3_FORCE_PATH_STYLE
-fi
-echo $DRYCC_STORAGE_BUCKET > BUCKET_NAME
diff --git a/rootfs/docker-entrypoint-initdb.d/002_create_bucket.sh b/rootfs/docker-entrypoint-initdb.d/002_create_bucket.sh
deleted file mode 100755
index 316af45..0000000
--- a/rootfs/docker-entrypoint-initdb.d/002_create_bucket.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-# ensure WAL log bucket exists
-envdir "$WALG_ENVDIR" create_bucket
diff --git a/rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh b/rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh
deleted file mode 100755
index 0f7f9f9..0000000
--- a/rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh
+++ /dev/null
@@ -1,71 +0,0 @@
-#!/usr/bin/env bash
-
-# check if there are any backups -- if so, let's restore
-# we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
-
-if [[ $(envdir "$WALG_ENVDIR" wal-g backup-list | wc -l) -gt "1" ]]; then
-  echo "Found backups. Restoring from backup..."
-  {
-    gosu postgres pg_ctl -D "$PGDATA" -w stop > /dev/null 2>&1
-  } || {
-    echo "ignore script errors"
-  }
-  rm -rf "$PGDATA"
-  envdir "$WALG_ENVDIR" wal-g backup-fetch "$PGDATA" LATEST
-  touch "$PGDATA/recovery.signal"
-  cat << EOF > "$PGDATA/postgresql.conf"
-# These settings are initialized by initdb, but they can be changed.
-log_timezone = 'UTC'
-lc_messages = 'C'     # locale for system error message
-lc_monetary = 'C'     # locale for monetary formatting
-lc_numeric = 'C'      # locale for number formatting
-lc_time = 'C'       # locale for time formatting
-default_text_search_config = 'pg_catalog.english'
-huge_pages = try
-wal_level = archive
-archive_mode = on
-archive_command = 'envdir "${WALG_ENVDIR}" wal-g wal-push %p'
-archive_timeout = 60
-listen_addresses = '*'
-max_connections = 1024
-restore_command = 'envdir "${WALG_ENVDIR}" wal-g wal-fetch \"%f\" \"%p\"'
-EOF
-  cat << EOF > "$PGDATA/pg_hba.conf"
-# "local" is for Unix domain socket connections only
-local   all             all                                     trust
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            trust
-# IPv6 local connections:
-host    all             all             ::1/128                 trust
-# IPv4 global connections
-host    all             all             0.0.0.0/0               md5
-EOF
-  touch "$PGDATA/pg_ident.conf"
-  touch "$PGDATA/recovery.signal"
-  chown -R postgres:postgres "$PGDATA"
-  chmod 0700 "$PGDATA"
-  gosu postgres pg_ctl -D "$PGDATA" \
-      -o "-c listen_addresses=''" \
-      -w start
-else
-  cat << EOF >> "$PGDATA/postgresql.conf"
-huge_pages = try
-wal_level = archive
-archive_mode = on
-archive_command = 'envdir "${WALG_ENVDIR}" wal-g wal-push %p'
-archive_timeout = 60
-max_connections = 1024
-EOF
-
-  # ensure $PGDATA has the right permissions
-  chown -R postgres:postgres "$PGDATA"
-  chmod 0700 "$PGDATA"
-
-  # reboot the server for wal_level to be set before backing up
-  echo "Rebooting postgres to enable archive mode"
-  gosu postgres pg_ctl -D "$PGDATA" -w restart
-fi
-
-# ensure $PGDATA has the right permissions
-chown -R postgres:postgres "$PGDATA"
-chmod 0700 "$PGDATA"
diff --git a/rootfs/docker-entrypoint-initdb.d/004_run_backups.sh b/rootfs/docker-entrypoint-initdb.d/004_run_backups.sh
deleted file mode 100755
index d311bd4..0000000
--- a/rootfs/docker-entrypoint-initdb.d/004_run_backups.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/usr/bin/env bash
-
-# Run periodic backups in the background
-gosu postgres backup &
diff --git a/rootfs/docker-entrypoint.sh b/rootfs/docker-entrypoint.sh
deleted file mode 100755
index d9de174..0000000
--- a/rootfs/docker-entrypoint.sh
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/bin/bash
-#
-# Originally copied and modified from
-# https://github.com/docker-library/postgres/blob/ec5ce80ca914e02c2d5eb9fde424039d4cee032e/9.4/docker-entrypoint.sh
-#
-set -e
-
-set_listen_addresses() {
-	sedEscapedValue="$(echo "$1" | sed 's/[\/&]/\\&/g')"
-	sed -ri "s/^#?(listen_addresses\s*=\s*)\S+/\1'$sedEscapedValue'/" "$PGDATA/postgresql.conf"
-}
-
-POSTGRES_USER="${DRYCC_DATABASE_USER}"
-POSTGRES_PASSWORD="${DRYCC_DATABASE_PASSWORD}"
-
-if [ "$1" = 'postgres' ]; then
-	mkdir -p "$PGDATA"
-	chmod 700 "$PGDATA"
-	chown -R postgres "$PGDATA"
-
-	chmod g+s /run/postgresql
-	chown -R postgres /run/postgresql
-
-	# look specifically for PG_VERSION, as it is expected in the DB dir
-	if [ ! -s "$PGDATA/PG_VERSION" ]; then
-		gosu postgres initdb
-		# check password first so we can output the warning before postgres
-		# messes it up
-		if [ "$POSTGRES_PASSWORD" ]; then
-			pass="PASSWORD '$POSTGRES_PASSWORD'"
-			authMethod=md5
-		else
-			# The - option suppresses leading tabs but *not* spaces. :)
-			cat >&2 <<-'EOWARN'
-				****************************************************
-				WARNING: No password has been set for the database.
-				         This will allow anyone with access to the
-				         Postgres port to access your database. In
-				         Docker's default configuration, this is
-				         effectively any other container on the same
-				         system.
-
-				         Use "-e POSTGRES_PASSWORD=password" to set
-				         it in "docker run".
-				****************************************************
-			EOWARN
-
-			pass=
-			authMethod=trust
-		fi
-
-		{ echo; echo "host all all 0.0.0.0/0 $authMethod"; } >> "$PGDATA/pg_hba.conf"
-
-		# internal start of server in order to allow set-up using psql-client
-		# does not listen on TCP/IP and waits until start finishes
-		gosu postgres pg_ctl -D "$PGDATA" \
-			-o "-c listen_addresses=''" \
-			-w start
-
-		: ${POSTGRES_USER:=postgres}
-		export POSTGRES_USER
-
-		if [ "$POSTGRES_USER" = 'postgres' ]; then
-			op='ALTER'
-		else
-			op='CREATE'
-		fi
-
-		psql --username postgres <<-EOSQL
-			$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
-		EOSQL
-		echo
-
-		# Initialize the passport and controller databases
-		psql --username postgres <<-EOSQL
-			CREATE DATABASE passport;
-			CREATE DATABASE controller;
-		EOSQL
-		echo
-
-		for f in /docker-entrypoint-initdb.d/*; do
-			case "$f" in
-				*.sh)  echo "$0: running $f"; . "$f" ;;
-				*.sql)
-					echo "$0: running $f";
-					psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < "$f"
-					echo
-					;;
-				*)     echo "$0: ignoring $f" ;;
-			esac
-			echo
-		done
-
-		gosu postgres pg_ctl -D "$PGDATA" -m fast -w stop
-		set_listen_addresses '*'
-
-		echo
-		echo 'PostgreSQL init process complete; ready for start up.'
-		echo
-	fi
-
-	exec gosu postgres "$@"
-fi
-
-exec "$@"
diff --git a/rootfs/entrypoint.sh b/rootfs/entrypoint.sh
new file mode 100755
index 0000000..40e2ddf
--- /dev/null
+++ b/rootfs/entrypoint.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+if [[ $UID -ge 10000 ]]; then
+    GID=$(id -g)
+    sed -e "s/^postgres:x:[^:]*:[^:]*:/postgres:x:$UID:$GID:/" /etc/passwd > /tmp/passwd
+    cat /tmp/passwd > /etc/passwd
+    rm /tmp/passwd
+fi
+
+cat > /data/patroni.yaml <<__EOF__
+bootstrap:
+  dcs:
+    postgresql:
+      use_pg_rewind: true
+  initdb:
+  - auth-host: md5
+  - auth-local: trust
+  - encoding: UTF8
+  - locale: ${LANG}
+  - data-checksums
+  pg_hba:
+  - host all all 0.0.0.0/0 md5
+  - host replication ${DRYCC_DATABASE_REPLICATOR} ${PATRONI_KUBERNETES_POD_IP}/16 md5
+  post_bootstrap: /usr/share/scripts/patroni/post_init.sh
+restapi:
+  connect_address: '${PATRONI_KUBERNETES_POD_IP}:8008'
+postgresql:
+  data_dir: '${PGDATA}'
+  connect_address: '${PATRONI_KUBERNETES_POD_IP}:5432'
+  authentication:
+    superuser:
+      username: '${DRYCC_DATABASE_SUPERUSER}'
+      password: '${DRYCC_DATABASE_SUPERUSER_PASSWORD}'
+    replication:
+      username: '${DRYCC_DATABASE_REPLICATOR}'
+      password: '${DRYCC_DATABASE_REPLICATOR_PASSWORD}'
+watchdog:
+  mode: off
+__EOF__
+
+unset DRYCC_DATABASE_SUPERUSER_PASSWORD DRYCC_DATABASE_REPLICATION_PASSWORD
+
+exec patroni /data/patroni.yaml
diff --git a/rootfs/usr/share/scripts/patroni/post_init.sh b/rootfs/usr/share/scripts/patroni/post_init.sh
new file mode 100755
index 0000000..90478b1
--- /dev/null
+++ b/rootfs/usr/share/scripts/patroni/post_init.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -Eeu
+
+if [[ ( -n "$DRYCC_DATABASE_USER") &&  ( -n "$DRYCC_DATABASE_PASSWORD")]]; then
+  echo "Creating user ${DRYCC_DATABASE_USER}"
+  psql "$1" -w -c "create user ${DRYCC_DATABASE_USER} WITH LOGIN ENCRYPTED PASSWORD '${DRYCC_DATABASE_PASSWORD}'"
+
+  echo "Creating passport and controller databases"
+  psql "$1" -w -c "CREATE DATABASE passport OWNER ${DRYCC_DATABASE_USER}"
+  psql "$1" -w -c "CREATE DATABASE controller OWNER ${DRYCC_DATABASE_USER}"
+
+else
+  echo "Skipping user creation"
+  echo "Skipping database creation"
+fi
