Merge pull request #79 from kmala/storage

feat(storage): Add support for multiple object storages

Author: kmala

contrib/ci/test.sh

@@ -32,8 +32,10 @@ echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-admin/acc
 # create fake AWS credentials for minio user credentials
 mkdir -p $CURRENT_DIR/tmp/aws-user
 # needs to be 20 characters long
+echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-user/accesskey
 echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-user/access-key-id
 # needs to be 40 characters long
+echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-user/secretkey
 echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-user/access-secret-key
 
 puts-step "creating fake kubernetes service account token"
@@ -47,7 +49,7 @@ echo "cert" > $CURRENT_DIR/tmp/k8s/ca.crt
 MINIO_JOB=$(docker run -dv $CURRENT_DIR/tmp/aws-admin:/var/run/secrets/deis/minio/admin -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/minio/user -v $CURRENT_DIR/tmp/k8s:/var/run/secrets/kubernetes.io/serviceaccount quay.io/deisci/minio:canary boot server /home/minio/)
 
 # boot postgres, linking the minio container and setting DEIS_MINIO_SERVICE_HOST and DEIS_MINIO_SERVICE_PORT
-PG_JOB=$(docker run -d --link $MINIO_JOB:minio -e BACKUP_FREQUENCY=1s -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/etc/wal-e.d/env $1)
+PG_JOB=$(docker run -d --link $MINIO_JOB:minio -e BACKUP_FREQUENCY=1s -e DATABASE_STORAGE=minio -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/objectstore/creds $1)
 
 # wait for postgres to boot
 puts-step "sleeping for 90s while postgres is booting..."

rootfs/Dockerfile

@@ -21,7 +21,7 @@ ENV LANG en_US.utf8
 RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
 
 ENV PG_MAJOR 9.4
-ENV PG_VERSION 9.4.6-1.pgdg80+1
+ENV PG_VERSION 9.4.7-1.pgdg80+1
 
 RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
 
@@ -48,6 +48,8 @@ RUN apt-get update && apt-get install -y \
 	pv \
 	python \
 	python-dev \
+	libssl-dev \
+	libffi-dev \
 	--no-install-recommends \
 	&& rm -rf /var/lib/apt/lists/*
 
@@ -56,7 +58,7 @@ RUN curl -sSL https://raw.githubusercontent.com/pypa/pip/7.1.2/contrib/get-pip.p
 
 # install deis/wal-e from the tip of the boto3-upgrade branch
 # NOTE (bacongobbler): we use the commit here so Docker's cache will bust when we make changes upstream
-RUN pip install --disable-pip-version-check --no-cache-dir git+https://github.com/deis/wal-e.git@dfa35c648b1a7b554122312d6c6b592be372e17a
+RUN pip install --disable-pip-version-check --no-cache-dir git+https://github.com/deis/wal-e.git@0eee12665149d8389cda9575b737359d066fa663
 
 # install python port of daemontools
 RUN pip install --disable-pip-version-check --no-cache-dir envdir

rootfs/bin/create_bucket

@@ -5,28 +5,69 @@ import sys
 
 import boto3
 import botocore
+import json
 from botocore.utils import fix_s3_host
+from botocore.client import Config
+from oauth2client.service_account import ServiceAccountCredentials
+from gcloud.storage.client import Client
+from gcloud import exceptions
+from azure.storage.blob import BlockBlobService
 
 bucket_name = os.getenv('BUCKET_NAME')
 
 if os.getenv('DATABASE_STORAGE') == "s3":
     conn = boto3.resource('s3')
-else:
-    conn = boto3.resource('s3', endpoint_url=os.getenv('S3_URL'))
-    # stop boto3 from automatically changing the endpoint
-    conn.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
-
-exists = True
-try:
-    conn.meta.client.head_bucket(Bucket=bucket_name)
-except botocore.exceptions.ClientError as e:
-    # If a client error is thrown, then check that it was a 404 error.
-    # If it was a 404 error, then the bucket does not exist.
-    error_code = int(e.response['Error']['Code'])
-    if error_code == 404:
+    exists = True
+    try:
+        conn.meta.client.head_bucket(Bucket=bucket_name)
+    except botocore.exceptions.ClientError as e:
+        # If a client error is thrown, then check that it was a 404 error.
+        # If it was a 404 error, then the bucket does not exist.
+        error_code = int(e.response['Error']['Code'])
+        if error_code == 404:
+            exists = False
+        else:
+            raise
+
+    if not exists:
+    	conn.create_bucket(Bucket=bucket_name)
+
+elif os.getenv('DATABASE_STORAGE') == "gcs":
+    scopes = ['https://www.googleapis.com/auth/devstorage.full_control']
+    credentials = ServiceAccountCredentials.from_json_keyfile_name(os.getenv('GS_APPLICATION_CREDS'), scopes=scopes)
+    with open(os.getenv('GS_APPLICATION_CREDS')) as data_file:
+        data = json.load(data_file)
+    client = Client(credentials=credentials, project=data['project_id'])
+    exists = True
+    try:
+        client.get_bucket(bucket_name)
+    except exceptions.NotFound:
         exists = False
-    else:
+    except:
         raise
+    if not exists:
+    	client.create_bucket(bucket_name)
+
+elif os.getenv('DATABASE_STORAGE') == "azure":
+  block_blob_service = BlockBlobService(account_name=os.getenv('WABS_ACCOUNT_NAME'), account_key=os.getenv('WABS_ACCESS_KEY'))
+  #It doesn't throw an exception if the container exists by default(https://github.com/Azure/azure-storage-python/blob/master/azure/storage/blob/baseblobservice.py#L504).
+  block_blob_service.create_container(bucket_name)
+
+else :
+  conn = boto3.resource('s3', endpoint_url=os.getenv('S3_URL'), config=Config(signature_version='s3v4'))
+  # stop boto3 from automatically changing the endpoint
+  conn.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
+  exists = True
+  try:
+      conn.meta.client.head_bucket(Bucket=bucket_name)
+  except botocore.exceptions.ClientError as e:
+      # If a client error is thrown, then check that it was a 404 error.
+      # If it was a 404 error, then the bucket does not exist.
+      error_code = int(e.response['Error']['Code'])
+      if error_code == 404:
+          exists = False
+      else:
+          raise
 
-if not exists:
-	conn.create_bucket(Bucket=bucket_name)
+  if not exists:
+  	conn.create_bucket(Bucket=bucket_name)

rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh

@@ -2,64 +2,42 @@
 
 cd "$WALE_ENVDIR"
 
-# access-key-id and access-secret-key files are mounted in via kubernetes secrets
-AWS_ACCESS_KEY_ID=$(cat access-key-id)
-AWS_SECRET_ACCESS_KEY=$(cat access-secret-key)
-AWS_DEFAULT_REGION="us-east-1"
-BUCKET_NAME="dbwal"
-
-if [ "$DATABASE_STORAGE" == "s3" ]; then
+if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
   AWS_ACCESS_KEY_ID=$(cat /var/run/secrets/deis/objectstore/creds/accesskey)
   AWS_SECRET_ACCESS_KEY=$(cat /var/run/secrets/deis/objectstore/creds/secretkey)
-  AWS_DEFAULT_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
-  BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
-else
-  # these only need to be set if we're not accessing S3 (boto will figure this out)
-  echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > WALE_S3_ENDPOINT
-  if [ "$DEIS_MINIO_SERVICE_PORT" == "80" ]; then
-    # If you add port 80 to the end of the endpoint_url, boto3 freaks out.
-    # God I hate boto3 some days.
-    echo "http://$DEIS_MINIO_SERVICE_HOST" > S3_URL
+  if [[ "$DATABASE_STORAGE" == "s3" ]]; then
+    AWS_DEFAULT_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
+    BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
   else
-    echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > S3_URL
+    # these only need to be set if we're not accessing S3 (boto will figure this out)
+    echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > WALE_S3_ENDPOINT
+    if [ "$DEIS_MINIO_SERVICE_PORT" == "80" ]; then
+      # If you add port 80 to the end of the endpoint_url, boto3 freaks out.
+      # God I hate boto3 some days.
+      echo "http://$DEIS_MINIO_SERVICE_HOST" > S3_URL
+    else
+      echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > S3_URL
+    fi
+    AWS_DEFAULT_REGION="us-east-1"
+    BUCKET_NAME="dbwal"
   fi
+  echo "s3://$BUCKET_NAME" > WALE_S3_PREFIX
+  echo $AWS_ACCESS_KEY_ID > AWS_ACCESS_KEY_ID
+  echo $AWS_SECRET_ACCESS_KEY > AWS_SECRET_ACCESS_KEY
+  echo $AWS_DEFAULT_REGION > AWS_DEFAULT_REGION
+  echo $BUCKET_NAME > BUCKET_NAME
+elif [ "$DATABASE_STORAGE" == "gcs" ]; then
+  GS_APPLICATION_CREDS="/var/run/secrets/deis/objectstore/creds/key.json"
+  BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
+  echo "gs://$BUCKET_NAME" > WALE_GS_PREFIX
+  echo $GS_APPLICATION_CREDS > GS_APPLICATION_CREDS
+  echo $BUCKET_NAME > BUCKET_NAME
+elif [ "$DATABASE_STORAGE" == "azure" ]; then
+  WABS_ACCOUNT_NAME=$(cat /var/run/secrets/deis/objectstore/creds/accountname)
+  WABS_ACCESS_KEY=$(cat /var/run/secrets/deis/objectstore/creds/accountkey)
+  BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-container)
+  echo $WABS_ACCOUNT_NAME > WABS_ACCOUNT_NAME
+  echo $WABS_ACCESS_KEY > WABS_ACCESS_KEY
+  echo "wabs://$BUCKET_NAME" > WALE_WABS_PREFIX
+  echo $BUCKET_NAME > BUCKET_NAME
 fi
-
-echo $AWS_ACCESS_KEY_ID > AWS_ACCESS_KEY_ID
-echo $AWS_SECRET_ACCESS_KEY > AWS_SECRET_ACCESS_KEY
-echo $AWS_DEFAULT_REGION > AWS_DEFAULT_REGION
-echo $BUCKET_NAME > BUCKET_NAME
-
-# setup envvars for wal-e
-echo "s3://$BUCKET_NAME" > WALE_S3_PREFIX
-
-
-# setup boto config
-mkdir -p /root/.aws /home/postgres/.aws
-
-cat << EOF > /root/.aws/credentials
-[default]
-aws_access_key_id = $AWS_ACCESS_KEY_ID
-aws_secret_access_key = $AWS_SECRET_ACCESS_KEY
-EOF
-
-if [ "$DATABASE_STORAGE" == "s3" ]; then
-  cat << EOF > /root/.aws/config
-[default]
-region = $AWS_DEFAULT_REGION
-EOF
-else
-  # HACK (bacongobbler): minio *must* use us-east-1 and signature version 4
-  # for authentication.
-  # see https://github.com/minio/minio#how-to-use-aws-cli-with-minio
-  cat << EOF > /root/.aws/config
-[default]
-region = $AWS_DEFAULT_REGION
-s3 =
-    signature_version = s3v4
-EOF
-fi
-
-# write AWS config to postgres homedir as well
-cp /root/.aws/* /home/postgres/.aws/
-chown -R postgres:postgres /home/postgres

rootfs/docker-entrypoint-initdb.d/003_restore_from_backup.sh

@@ -15,7 +15,7 @@ gosu postgres pg_ctl -D "$PGDATA" -w restart
 # check if there are any backups -- if so, let's restore
 # we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
 if [[ $(envdir "$WALE_ENVDIR" wal-e --terse backup-list | wc -l) -gt "1" ]]; then
-  echo "Found backups on S3. Restoring from backup..."
+  echo "Found backups. Restoring from backup..."
   gosu postgres pg_ctl -D "$PGDATA" -w stop
   rm -rf "$PGDATA"
   envdir "$WALE_ENVDIR" wal-e backup-fetch "$PGDATA" LATEST
@@ -44,6 +44,8 @@ host    all             all             0.0.0.0/0               md5
 EOF
   touch "$PGDATA/pg_ident.conf"
   echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" >> "$PGDATA/recovery.conf"
+  chown -R postgres:postgres "$PGDATA"
+  chmod 0700 "$PGDATA"
   gosu postgres pg_ctl -D "$PGDATA" \
       -o "-c listen_addresses=''" \
       -t 1200 \