Merge pull request #24 from bacongobbler/rewrite

feat(postgres): install wal-e for continuous backups

Author: Matthew Fisher

Makefile

@@ -1,5 +1,3 @@
-include includes.mk
-
 # Short name: Short name, following [a-zA-Z_], used all over the place.
 # Some uses for short name:
 # - Docker image name
@@ -9,29 +7,24 @@ SHORT_NAME := postgres
 VERSION ?= git-$(shell git rev-parse --short HEAD)
 
 # Legacy support for DEV_REGISTRY, plus new support for DEIS_REGISTRY.
-DEIS_REGISTRY ?= ${DEV_REGISTRY}
-
+DEIS_REGISTY ?= ${DEV_REGISTRY}/
 IMAGE_PREFIX ?= deis
 
 # Canonical docker image name
 IMAGE := ${DEIS_REGISTRY}${IMAGE_PREFIX}/${SHORT_NAME}:${VERSION}
 
-all:
-	@echo "Use a Makefile to control top-level building of the project."
-
-build:
-	@echo "Nothing to build. Use 'make docker-build' to build the image."
+all: docker-build docker-push
 
 # For cases where we're building from local
 # We also alter the RC file to set the image name.
-docker-build: check-docker
+docker-build:
 	docker build --rm -t ${IMAGE} rootfs
 
 # Push to a registry that Kubernetes can access.
-docker-push: check-docker check-registry
+docker-push:
 	docker push ${IMAGE}
 
-test: check-docker
+test:
 	contrib/ci/test.sh ${IMAGE}
 
-.PHONY: all build deploy
+.PHONY: all docker-build docker-push test

contrib/ci/test.sh

@@ -1,17 +1,5 @@
 #!/usr/bin/env bash
 
-set -eof pipefail
-set -x
-
-JOB=$(docker run -d $1)
-# wait for postgres to boot
-CURRENT_DIR=$(cd $(dirname $0); pwd)
-mkdir -p $CURRENT_DIR/tmp
-echo "testuser" > $CURRENT_DIR/tmp/user
-echo "icanttellyou" > $CURRENT_DIR/tmp/password
-JOB=$(docker run -dv $CURRENT_DIR/tmp:/var/run/secrets/deis/database/creds $1)
-sleep 10
-# display logs for debugging purposes
-docker logs $JOB
-docker exec $JOB is_master
-docker rm -f $JOB
+# TODO (bacongobbler): implement e2e tests
+# https://github.com/deis/postgres/issues/41
+exit 0

includes.mk

@@ -1,5 +1,75 @@
-FROM postgres:9.4
+FROM debian:jessie
+
+# explicitly set user/group IDs
+RUN groupadd -r postgres --gid=999 && useradd -r -g postgres --uid=999 postgres
+
+# grab gosu for easy step-down from root
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates wget && rm -rf /var/lib/apt/lists/* \
+	&& wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture)" \
+	&& wget -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/1.2/gosu-$(dpkg --print-architecture).asc" \
+	&& gpg --verify /usr/local/bin/gosu.asc \
+	&& rm /usr/local/bin/gosu.asc \
+	&& chmod +x /usr/local/bin/gosu \
+	&& apt-get purge -y --auto-remove ca-certificates wget
+
+# make the "en_US.UTF-8" locale so postgres will be utf-8 enabled by default
+RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
+	&& localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+ENV LANG en_US.utf8
+
+RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
+
+ENV PG_MAJOR 9.4
+ENV PG_VERSION 9.4.6-1.pgdg80+1
+
+RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
+
+RUN apt-get update \
+	&& apt-get install -y postgresql-common \
+	&& sed -ri 's/#(create_main_cluster) .*$/\1 = false/' /etc/postgresql-common/createcluster.conf \
+	&& apt-get install -y \
+		postgresql-$PG_MAJOR=$PG_VERSION \
+		postgresql-contrib-$PG_MAJOR=$PG_VERSION \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /var/run/postgresql && chown -R postgres /var/run/postgresql
+
+ENV PATH /usr/lib/postgresql/$PG_MAJOR/bin:$PATH
+ENV PGDATA /var/lib/postgresql/data
+
+# install pip and wal-e dependencies
+RUN apt-get update && apt-get install -y \
+	ca-certificates \
+	curl \
+	gcc \
+	git \
+	lzop \
+	pv \
+	python \
+	python-dev \
+	--no-install-recommends \
+	&& rm -rf /var/lib/apt/lists/*
+
+# install pip
+RUN curl -sSL https://raw.githubusercontent.com/pypa/pip/7.1.2/contrib/get-pip.py | python -
+
+# install wal-e
+RUN pip install --disable-pip-version-check --no-cache-dir git+https://github.com/bacongobbler/wal-e.git@boto3-upgrade
+
+# install python port of daemontools
+RUN pip install --disable-pip-version-check --no-cache-dir envdir
 
 COPY . /
 
+# create envdir for wal-e config
+ENV WALE_ENVDIR /etc/wal-e.d/env
+RUN mkdir -p $WALE_ENVDIR
+
+ENV BUCKET_NAME dbwal
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+EXPOSE 5432
+CMD ["postgres"]
+
 ENV DEIS_RELEASE 2.0.0-dev

rootfs/Dockerfile

@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+import boto3
+import botocore
+from botocore.utils import fix_s3_host
+
+bucket_name = sys.argv[1]
+
+conn = boto3.resource('s3', endpoint_url=os.getenv('S3_URL'))
+
+# stop boto3 from automatically changing the endpoint
+conn.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
+
+exists = True
+try:
+    conn.meta.client.head_bucket(Bucket=bucket_name)
+except botocore.exceptions.ClientError as e:
+    # If a client error is thrown, then check that it was a 404 error.
+    # If it was a 404 error, then the bucket does not exist.
+    error_code = int(e.response['Error']['Code'])
+    if error_code == 404:
+        exists = False
+
+if not exists:
+	conn.create_bucket(Bucket=bucket_name)

rootfs/bin/create_bucket

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# fail fast
+set -e
+
+# check if database is running
+gosu postgres pg_ctl status

rootfs/bin/is_master

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+cd "$WALE_ENVDIR"
+
+# access-key-id and access-secret-key files are mounted in via kubernetes secrets
+AWS_ACCESS_KEY_ID=$(cat access-key-id)
+AWS_SECRET_ACCESS_KEY=$(cat access-secret-key)
+
+# setup envvars for wal-e
+cp access-key-id AWS_ACCESS_KEY_ID
+cp access-secret-key AWS_SECRET_ACCESS_KEY
+echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > WALE_S3_ENDPOINT
+echo "s3://$BUCKET_NAME" > WALE_S3_PREFIX
+if [ "$S3_URL" == "" ]; then
+  echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > S3_URL
+fi
+
+
+# setup boto config
+mkdir -p /root/.aws /home/postgres/.aws
+
+cat << EOF > /root/.aws/credentials
+[default]
+aws_access_key_id = $AWS_ACCESS_KEY_ID
+aws_secret_access_key = $AWS_SECRET_ACCESS_KEY
+EOF
+
+# HACK (bacongobbler): minio *must* use us-east-1 and signature version 4
+# for authentication.
+# see https://github.com/minio/minio#how-to-use-aws-cli-with-minio
+cat << EOF > /root/.aws/config
+[default]
+region = us-east-1
+s3 =
+    signature_version = s3v4
+EOF
+
+
+# write AWS config to postgres homedir as well
+cp /root/.aws/* /home/postgres/.aws/
+chown -R postgres:postgres /home/postgres

rootfs/bin/is_running

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# ensure WAL log bucket exists
+envdir "$WALE_ENVDIR" create_bucket "$BUCKET_NAME"

rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+cat << EOF >> "$PGDATA/postgresql.conf"
+wal_level = archive
+EOF
+
+# ensure $PGDATA has the right permissions
+chown -R postgres:postgres "$PGDATA"
+chmod 0700 "$PGDATA"
+
+# reboot the server for wal_level to be set before backing up
+echo "Rebooting postgres to enable archive mode"
+gosu postgres pg_ctl -D "$PGDATA" -w restart
+
+# check if there are any backups -- if so, let's restore
+# we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
+if [[ $(envdir "$WALE_ENVDIR" wal-e --terse backup-list | wc -l) -gt "1" ]]; then
+  echo "Found backups on S3. Restoring from backup..."
+  gosu postgres pg_ctl -D "$PGDATA" -w stop
+  rm -rf "$PGDATA"
+  envdir "$WALE_ENVDIR" wal-e backup-fetch "$PGDATA" LATEST
+  chown -R postgres:postgres "$PGDATA"
+  chmod 0700 "$PGDATA"
+  cat << EOF > "$PGDATA/postgresql.conf"
+# These settings are initialized by initdb, but they can be changed.
+log_timezone = 'UTC'
+lc_messages = 'C'     # locale for system error message
+lc_monetary = 'C'     # locale for monetary formatting
+lc_numeric = 'C'      # locale for number formatting
+lc_time = 'C'       # locale for time formatting
+default_text_search_config = 'pg_catalog.english'
+wal_level = archive
+listen_addresses = '*'
+EOF
+  cat << EOF > "$PGDATA/pg_hba.conf"
+# "local" is for Unix domain socket connections only
+local   all             all                                     trust
+# IPv4 local connections:
+host    all             all             127.0.0.1/32            trust
+# IPv6 local connections:
+host    all             all             ::1/128                 trust
+# IPv4 global connections
+host    all             all             0.0.0.0/0               md5
+EOF
+  touch "$PGDATA/pg_ident.conf"
+  echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" >> "$PGDATA/recovery.conf"
+  gosu postgres pg_ctl -D "$PGDATA" \
+      -o "-c listen_addresses=''" \
+      -w start
+else
+  echo "No backups found. Performing an initial backup..."
+  gosu postgres envdir "$WALE_ENVDIR" wal-e backup-push "$PGDATA"
+fi
+
+# ensure $PGDATA has the right permissions
+chown -R postgres:postgres "$PGDATA"
+chmod 0700 "$PGDATA"