fix(database): run backups in the background

Instead of running the backup in the publish loop, we can schedule the
backups on a timed interval. I tried to do this with cron, but I was
having trouble sending the output of the forked processes to
/dev/stdout, so instead I just forked a for loop in the background :)

Author: Matthew Fisher

database/bin/backup

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-3h}
+export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
+
+while true; do
+    sleep "$BACKUP_FREQUENCY"
+    echo "database: performing a backup..."
+    if [[ -f /var/lib/postgresql/9.3/main/recovery.conf ]] ; then
+        echo "database: database is currently recovering from a backup. Will try again next loop..."
+    else
+        # perform a backup
+        envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
+        # only retain the latest BACKUPS_TO_RETAIN backups
+        envdir /etc/wal-e.d/env wal-e delete --confirm retain $BACKUPS_TO_RETAIN
+        echo "database: backup has been completed."
+    fi
+done

database/bin/boot

@@ -16,10 +16,6 @@ export ETCD_PATH=${ETCD_PATH:-/deis/database}
 export ETCD_TTL=${ETCD_TTL:-20}
 
 export BUCKET_NAME=${BUCKET_NAME:-db_wal}
-export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
-
-# how many TTL/2 sleeps between backups -- 2160 is 3 hours
-export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-2160}
 
 # wait for etcd to be available
 until etcdctl --no-sync -C $ETCD ls >/dev/null 2>&1; do
@@ -125,9 +121,10 @@ if [[ "${initial_backup}" == "1" ]] ; then
   sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
 fi
 
+sudo -Eu postgres /app/bin/backup &
+
 echo "database: postgres is running..."
 
-count=1
 # publish the service to etcd using the injected HOST and EXTERNAL_PORT
 if [[ ! -z $EXTERNAL_PORT ]]; then
 	# configure service discovery
@@ -143,23 +140,6 @@ if [[ ! -z $EXTERNAL_PORT ]]; then
 	while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
 		etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
 		etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $EXTERNAL_PORT --ttl $ETCD_TTL >/dev/null
-
-    # perform a backup whenever we hit BACKUP_FREQUENCY
-    # we really need cron :(
-    if [[ "${count}" -ge "${BACKUP_FREQUENCY}" ]] ; then
-      echo "database: performing a backup..."
-      if [[ -f /var/lib/postgresql/9.3/main/recovery.conf ]] ; then
-        echo "database: database is currently recovering from a backup. Will try again next loop..."
-      else
-        # perform a backup
-        sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
-        # only retain the latest BACKUPS_TO_RETAIN backups
-        sudo -u postgres envdir /etc/wal-e.d/env wal-e delete --confirm retain ${BACKUPS_TO_RETAIN}
-        count=0
-        echo "database: backup has been completed."
-      fi
-    fi
-    ((count++))
 		sleep $(($ETCD_TTL/2)) # sleep for half the TTL
 	done
 

database/tests/recovery_test.go

@@ -107,13 +107,13 @@ func TestDatabaseRecovery(t *testing.T) {
 			"-e", "HOST="+host,
 			"-e", "ETCD_PORT="+etcdPort,
 			"-e", "ETCD_TTL=2",
-			"-e", "BACKUP_FREQUENCY=0",
+			"-e", "BACKUP_FREQUENCY=1s",
 			"-e", "BACKUPS_TO_RETAIN=100",
 			imageName)
 	}
 
 	stopDatabase := func() {
-		fmt.Print("--- Stopping data-database... ")
+		fmt.Println("--- Stopping data-database... ")
 		if err = stdout.Close(); err != nil {
 			t.Fatal("Failed to closeStdout")
 		}
@@ -124,7 +124,7 @@ func TestDatabaseRecovery(t *testing.T) {
 	//ACTION
 
 	//STEP 1: start db with volume A and wait for init to complete
-	fmt.Print("--- Starting database with Volume A... ")
+	fmt.Println("--- Starting database with Volume A... ")
 	go startDatabase(databaseVolumeA)
 	dockercli.WaitForLine(t, stdout, "database: postgres is running...", true)
 	fmt.Println("Done")
@@ -148,7 +148,7 @@ func TestDatabaseRecovery(t *testing.T) {
 	execSql(t, db, "create table api_foo(t text)")
 
 	//STEP 2b: make sure we observed full backup cycle after forced checkpoint
-	fmt.Print("--- Waiting for the change to be backed up... ")
+	fmt.Println("--- Waiting for the change to be backed up... ")
 	dockercli.WaitForLine(t, stdout, "database: performing a backup...", true)
 	dockercli.WaitForLine(t, stdout, "database: backup has been completed.", true)
 	fmt.Println("Done")