Merge pull request #1910 from carmstrong/deis-store

feat(store): add deis-store component

Author: Gabriel Monroy

Makefile

@@ -4,8 +4,8 @@
 
 include includes.mk
 
-COMPONENTS=builder cache controller database logger logspout publisher registry router
-START_ORDER=publisher logger logspout database cache registry controller builder router
+COMPONENTS=builder cache controller database logger logspout publisher registry router store
+START_ORDER=publisher store logger logspout database cache registry controller builder router
 CLIENTS=client deisctl
 
 all: build run

builder/tests/builder_test.go

@@ -41,9 +41,9 @@ func TestBuilder(t *testing.T) {
 	host, port := utils.HostAddress(), utils.RandomPort()
 	fmt.Printf("--- Run deis/builder:%s at %s:%s\n", tag, host, port)
 	name := "deis-builder-" + tag
-	defer cli.CmdRm("-f", name)
+	defer cli.CmdRm("-f", "-v", name)
 	go func() {
-		_ = cli.CmdRm("-f", name)
+		_ = cli.CmdRm("-f", "-v", name)
 		err = dockercli.RunContainer(cli,
 			"--name", name,
 			"--rm",

contrib/docker-registry/user-data

@@ -2,13 +2,37 @@
 ---
 coreos:
   units:
-  - name: docker-registry.service
-    command: start
-    content: |
-      [Unit]
-      Description=Docker Registry server
-
-      [Service]
-      TimeoutStartSec=10min
-      ExecStartPre=/bin/sh -c "docker history registry:latest >/dev/null || docker pull registry:latest"
-      ExecStart=/usr/bin/docker run --name docker-registry -e STORAGE_PATH=/registry -e SEARCH_BACKEND=sqlalchemy -p 5000:5000 registry:latest
+    - name: stop-docker-socket.service
+      command: start
+      content: |
+        [Unit]
+        Description=stop docker.socket
+
+        [Service]
+        Type=oneshot
+        ExecStart=/usr/bin/systemctl stop docker.socket
+        ExecStartPost=/usr/bin/systemctl mask docker.socket
+    - name: docker-registry.service
+      command: start
+      content: |
+        [Unit]
+        Description=Docker Registry server
+
+        [Service]
+        TimeoutStartSec=10min
+        ExecStartPre=/bin/sh -c "docker history registry:latest >/dev/null || docker pull registry:latest"
+        ExecStart=/usr/bin/docker run --name docker-registry -e STORAGE_PATH=/registry -e SEARCH_BACKEND=sqlalchemy -p 5000:5000 registry:latest
+    - name: docker-tcp.socket
+      command: start
+      enable: true
+      content: |
+        [Unit]
+        Description=Docker Socket for the API
+
+        [Socket]
+        ListenStream=2375
+        Service=docker.service
+        BindIPv6Only=both
+
+        [Install]
+        WantedBy=sockets.target

contrib/gce/README.md

@@ -321,11 +321,9 @@ UNIT                        MACHINE                     LOAD    ACTIVE  SUB
 deis-builder@1.service      dea53588.../172.17.8.100    loaded  active  running
 deis-cache@1.service        dea53588.../172.17.8.100    loaded  active  running
 deis-controller@1.service   dea53588.../172.17.8.100    loaded  active  running
-deis-database-data.service  dea53588.../172.17.8.100    loaded  active  exited
 deis-database@1.service     dea53588.../172.17.8.100    loaded  active  running
 deis-logger-data.service    dea53588.../172.17.8.100    loaded  active  exited
 deis-logger@1.service       dea53588.../172.17.8.100    loaded  active  running
-deis-registry-data.service  dea53588.../172.17.8.100    loaded  active  exited
 deis-registry@1.service     dea53588.../172.17.8.100    loaded  active  running
 deis-router@1.service       dea53588.../172.17.8.100    loaded  active  running
 ```

database/Dockerfile

@@ -13,13 +13,37 @@ RUN curl -sSL -o /usr/local/bin/etcdctl https://s3-us-west-2.amazonaws.com/opdem
 RUN curl -sSL -o /usr/local/bin/confd https://s3-us-west-2.amazonaws.com/opdemand/confd-v0.5.0-json \
     && chmod +x /usr/local/bin/confd
 
-# install 9.3 from postgresql.org repository
+# install postgresql 9.3 from postgresql.org repository as well as requirements for building wal-e
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 RUN curl -sk https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-RUN apt-get update && apt-get install -yq postgresql-9.3 && /etc/init.d/postgresql stop
+RUN apt-get update && apt-get install -yq \
+                                      curl \
+                                      daemontools \
+                                      file \
+                                      gcc \
+                                      git \
+                                      libxml2-dev \
+                                      libxslt1-dev \
+                                      lzop \
+                                      postgresql-9.3 \
+                                      pv \
+                                      python-dev
+
+RUN /etc/init.d/postgresql stop
+
+# install pip
+RUN curl -sSL https://raw.githubusercontent.com/pypa/pip/1.5.6/contrib/get-pip.py | python -
+
+# install wal-e
+WORKDIR /tmp
+RUN git clone https://github.com/wal-e/wal-e.git
+WORKDIR /tmp/wal-e
+RUN git checkout c6f0d02
+RUN pip install .
+RUN mkdir -p /etc/wal-e.d/env
+RUN chown -R root:postgres /etc/wal-e.d
 
 # define the execution environment
-VOLUME ["/var/lib/postgresql"]
 WORKDIR /app
 CMD ["/app/bin/boot"]
 EXPOSE 5432

database/bin/boot

@@ -15,18 +15,15 @@ export ETCD="$HOST:$ETCD_PORT"
 export ETCD_PATH=${ETCD_PATH:-/deis/database}
 export ETCD_TTL=${ETCD_TTL:-10}
 
-# initialize data volume
-if [[ ! -d /var/lib/postgresql/9.3/main ]]; then
-  mkdir -p /var/lib/postgresql/9.3
-  chown -R postgres:postgres /var/lib/postgresql
-  sudo -u postgres /usr/lib/postgresql/9.3/bin/initdb -D /var/lib/postgresql/9.3/main
-else
-  chown -R postgres:postgres /var/lib/postgresql
-fi
+export BUCKET_NAME=${BUCKET_NAME:-db_wal}
+export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
+
+# how many TTL/2 sleeps between backups -- 2160 is 3 hours
+export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-2160}
 
 # wait for etcd to be available
 until etcdctl --no-sync -C $ETCD ls >/dev/null 2>&1; do
-	echo "waiting for etcd at $ETCD..."
+	echo "database: waiting for etcd at $ETCD..."
 	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
 done
 
@@ -43,13 +40,39 @@ etcd_set_default adminPass ${PG_ADMIN_PASS:-changeme123}
 etcd_set_default user ${PG_USER_NAME:-deis}
 etcd_set_default password ${PG_USER_PASS:-changeme123}
 etcd_set_default name ${PG_USER_DB:-deis}
+etcd_set_default bucketName ${BUCKET_NAME}
 
 # wait for confd to run once and install initial templates
 until confd -onetime -node $ETCD -config-file /app/confd.toml; do
 	echo "database: waiting for confd to write initial templates..."
 	sleep $(($ETCD_TTL/2))  # sleep for half the TTL
 done
 
+# ensure WAL log bucket exists
+envdir /etc/wal-e.d/env /app/bin/create_bucket ${BUCKET_NAME}
+
+initial_backup=0
+if [[ ! -f /var/lib/postgresql/9.3/main/initialized ]]; then
+  echo "database: no existing database found."
+  # check if there are any backups -- if so, let's restore
+  # we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
+  if [[ `envdir /etc/wal-e.d/env wal-e --terse backup-list | wc -l` -gt "1" ]]; then
+    echo "database: restoring from backup..."
+    rm -rf /var/lib/postgresql/9.3/main
+    sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-fetch /var/lib/postgresql/9.3/main LATEST
+    chown -R postgres:postgres /var/lib/postgresql/9.3/main
+    chmod 0700 /var/lib/postgresql/9.3/main
+    echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" | sudo -u postgres tee /var/lib/postgresql/9.3/main/recovery.conf >/dev/null
+  else
+    echo "database: no backups found. Initializing a new database..."
+    initial_backup=1
+  fi
+  # either way, we mark the database as initialized
+  touch /var/lib/postgresql/9.3/main/initialized
+else
+  echo "database: existing data directory found. Starting postgres..."
+fi
+
 # run the service in the background
 sudo -i -u postgres /usr/lib/postgresql/9.3/bin/postgres \
                     -c config-file=${PG_CONFIG:-/etc/postgresql/9.3/main/postgresql.conf} \
@@ -75,11 +98,17 @@ sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".5432\
 # perform a one-time reload to populate database entries
 /usr/local/bin/reload
 
-echo deis-database running...
+if [[ "${initial_backup}" == "1" ]] ; then
+  echo "database: performing an initial backup..."
+  # perform an initial backup
+  sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
+fi
+
+echo "database: postgres is running..."
 
+count=1
 # publish the service to etcd using the injected HOST and EXTERNAL_PORT
 if [[ ! -z $EXTERNAL_PORT ]]; then
-
 	# configure service discovery
 	PORT=${PORT:-5432}
 	PROTO=${PROTO:-tcp}
@@ -93,6 +122,22 @@ if [[ ! -z $EXTERNAL_PORT ]]; then
 	while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
 		etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
 		etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $EXTERNAL_PORT --ttl $ETCD_TTL >/dev/null
+
+    # perform a backup whenever we hit BACKUP_FREQUENCY
+    # we really need cron :(
+    if [[ "${count}" -ge "${BACKUP_FREQUENCY}" ]] ; then
+      echo "database: performing a backup..."
+      if [[ -f /var/lib/postgresql/9.3/main/recovery.conf ]] ; then
+        echo "database: database is currently recovering from a backup. Will try again next loop..."
+      else
+        # perform a backup
+        sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
+        # only retain the latest BACKUPS_TO_RETAIN backups
+        sudo -u postgres envdir /etc/wal-e.d/env wal-e delete --confirm retain ${BACKUPS_TO_RETAIN}
+        count=0
+      fi
+    fi
+    ((count++))
 		sleep $(($ETCD_TTL/2)) # sleep for half the TTL
 	done
 

database/conf.d/AWS_ACCESS_KEY_ID.toml

@@ -0,0 +1,10 @@
+[template]
+src   = "AWS_ACCESS_KEY_ID"
+dest  = "/etc/wal-e.d/env/AWS_ACCESS_KEY_ID"
+uid = 0
+gid = 106
+mode  = "0640"
+keys = [
+  "/deis/store/gateway",
+]
+check_cmd = "/app/bin/check {{ .src }}"

database/conf.d/AWS_SECRET_ACCESS_KEY.toml

@@ -0,0 +1,10 @@
+[template]
+src   = "AWS_SECRET_ACCESS_KEY"
+dest  = "/etc/wal-e.d/env/AWS_SECRET_ACCESS_KEY"
+uid = 0
+gid = 106
+mode  = "0640"
+keys = [
+  "/deis/store/gateway",
+]
+check_cmd = "/app/bin/check {{ .src }}"

database/conf.d/WALE_S3_ENDPOINT.toml

@@ -0,0 +1,10 @@
+[template]
+src   = "WALE_S3_ENDPOINT"
+dest  = "/etc/wal-e.d/env/WALE_S3_ENDPOINT"
+uid = 0
+gid = 106
+mode  = "0640"
+keys = [
+  "/deis/store/gateway",
+]
+check_cmd = "/app/bin/check {{ .src }}"

database/conf.d/WALE_S3_PREFIX.toml

@@ -0,0 +1,11 @@
+[template]
+src   = "WALE_S3_PREFIX"
+dest  = "/etc/wal-e.d/env/WALE_S3_PREFIX"
+uid = 0
+gid = 106
+mode  = "0640"
+keys = [
+  "/deis/store/gateway",
+  "/deis/database",
+]
+check_cmd = "/app/bin/check {{ .src }}"