feat(postgres): run periodic backups in the background

backup retention and frequency of backups are configurable through
environment variables.

Author: Matthew Fisher

contrib/ci/test.sh

@@ -6,6 +6,10 @@ puts-step() {
   echo "-----> $@"
 }
 
+puts-error() {
+  echo "!!!    $@"
+}
+
 # make sure we are in this dir
 CURRENT_DIR=$(cd $(dirname $0); pwd)
 
@@ -43,11 +47,11 @@ echo "cert" > $CURRENT_DIR/tmp/k8s/ca.crt
 MINIO_JOB=$(docker run -dv $CURRENT_DIR/tmp/aws-admin:/var/run/secrets/deis/minio/admin -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/minio/user -v $CURRENT_DIR/tmp/k8s:/var/run/secrets/kubernetes.io/serviceaccount quay.io/deisci/minio:v2-beta boot server /home/minio/)
 
 # boot postgres, linking the minio container and setting DEIS_MINIO_SERVICE_HOST and DEIS_MINIO_SERVICE_PORT
-PG_JOB=$(docker run -d --link $MINIO_JOB:minio -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/etc/wal-e.d/env $1)
+PG_JOB=$(docker run -d --link $MINIO_JOB:minio -e BACKUP_FREQUENCY=1s -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/etc/wal-e.d/env $1)
 
 # wait for postgres to boot
-puts-step "sleeping for 1 minute while postgres is booting..."
-sleep 1m
+puts-step "sleeping for 90s while postgres is booting..."
+sleep 90s
 
 # display logs for debugging purposes
 puts-step "displaying minio logs"
@@ -59,6 +63,14 @@ docker logs $PG_JOB
 puts-step "checking if database is running"
 docker exec $PG_JOB is_running
 
+# check if minio has the 5 backups
+puts-step "checking if minio has 5 backups"
+NUM_BACKUPS="$(docker exec $MINIO_JOB ls /home/minio/dbwal/basebackups_005/ | grep json | wc -l)"
+if [[ ! "$NUM_BACKUPS" -eq "5" ]]; then
+  puts-error "did not find 5 base backups, which is the default (found $NUM_BACKUPS)"
+  exit 1
+fi
+
 # success, kill off jobs
 puts-step "destroying containers"
 docker rm -f $MINIO_JOB

rootfs/bin/backup

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-12h}
+export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
+
+while true; do
+    sleep "$BACKUP_FREQUENCY"
+    echo "Performing a base backup..."
+    if [[ -f "$PGDATA/recovery.conf" ]] ; then
+        echo "Database is currently recovering from a backup. Will try again next loop..."
+    else
+        # perform a backup
+        envdir "$WALE_ENVDIR" wal-e backup-push "$PGDATA"
+        # only retain the latest BACKUPS_TO_RETAIN backups
+        envdir "$WALE_ENVDIR" wal-e delete --confirm retain "$BACKUPS_TO_RETAIN"
+        echo "Backup has been completed."
+    fi
+done

rootfs/docker-entrypoint-initdb.d/005_run_backups.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Run periodic backups in the background
+gosu postgres backup &