chore(Dockerfile): bump to wal-e v1.0.1

See https://github.com/wal-e/wal-e/releases/tag/v1.0.1

Author: Matthew Fisher

Dockerfile

@@ -15,7 +15,7 @@ RUN adduser --system \
     --group \
     postgres
 
-RUN buildDeps='gcc git libffi-dev libssl-dev python-dev python-pip python-wheel' && \
+RUN buildDeps='gcc git libffi-dev libssl-dev python3-dev python3-pip python3-wheel' && \
     localedef -i en_US -c -f UTF-8 -A /etc/locale.alias en_US.UTF-8 && \
     export DEBIAN_FRONTEND=noninteractive && \
     apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 && \
@@ -28,17 +28,21 @@ RUN buildDeps='gcc git libffi-dev libssl-dev python-dev python-pip python-wheel'
         postgresql-$PG_MAJOR=$PG_VERSION \
         postgresql-contrib-$PG_MAJOR=$PG_VERSION \
         pv \
-        python \
+        python3 \
         postgresql-common \
         util-linux \
         # swift package needs pkg_resources and setuptools
-        python-pkg-resources \
-        python-setuptools && \
+        python3-pkg-resources \
+        python3-setuptools && \
+    ln -sf /usr/bin/python3 /usr/bin/python && \
+    ln -sf /usr/bin/pip3 /usr/bin/pip && \
     mkdir -p /run/postgresql && \
     chown -R postgres /run/postgresql && \
-    pip install --disable-pip-version-check --no-cache-dir git+https://github.com/deis/wal-e.git@380821a6c4ea4f98a244680d7c6c5b04b8c694b3 \
-                                                           google-gax===0.12.5 \
-                                                           envdir && \
+    pip install --disable-pip-version-check --no-cache-dir \
+        envdir==0.7 \
+        wal-e[aws,azure,google,swift]==v1.0.1 && \
+    # "upgrade" boto to 2.42.0 + the patch to fix minio connections
+    pip install --disable-pip-version-check --no-cache-dir --upgrade git+https://github.com/deis/boto@c1e71a737bef48f934b6108c06a84e483173a2da && \
     # cleanup
     apt-get purge -y --auto-remove $buildDeps && \
     apt-get autoremove -y && \

rootfs/bin/create_bucket

@@ -2,56 +2,50 @@
 
 import os
 
-import boto3
-import botocore
+import boto
 import json
 import swiftclient
-from botocore.utils import fix_s3_host
-from botocore.client import Config
+from boto import config as botoconfig
+from boto.s3.connection import S3Connection, OrdinaryCallingFormat
 from oauth2client.service_account import ServiceAccountCredentials
 from gcloud.storage.client import Client
 from gcloud import exceptions
-from azure.storage.blob import BlockBlobService
+from azure.storage.blob import BlobService
+
+def bucket_exists(conn, name):
+    bucket = conn.lookup(name)
+    if not bucket:
+        return False
+    return True
 
 bucket_name = os.getenv('BUCKET_NAME')
 
 if os.getenv('DATABASE_STORAGE') == "s3":
-    conn = boto3.resource('s3')
-    exists = True
-    try:
-        conn.meta.client.head_bucket(Bucket=bucket_name)
-    except botocore.exceptions.ClientError as e:
-        # If a client error is thrown, then check that it was a 404 error.
-        # If it was a 404 error, then the bucket does not exist.
-        error_code = int(e.response['Error']['Code'])
-        if error_code == 404:
-            exists = False
-        else:
-            raise
+    conn = boto.connect_s3()
 
-    if not exists:
-        conn.create_bucket(Bucket=bucket_name)
+    if not bucket_exists(conn, bucket_name):
+        conn.create_bucket(bucket_name)
 
 elif os.getenv('DATABASE_STORAGE') == "gcs":
     scopes = ['https://www.googleapis.com/auth/devstorage.full_control']
     credentials = ServiceAccountCredentials.from_json_keyfile_name(os.getenv('GS_APPLICATION_CREDS'), scopes=scopes)
     with open(os.getenv('GS_APPLICATION_CREDS')) as data_file:
         data = json.load(data_file)
-    client = Client(credentials=credentials, project=data['project_id'])
+    conn = Client(credentials=credentials, project=data['project_id'])
     exists = True
     try:
-        client.get_bucket(bucket_name)
+        conn.get_bucket(bucket_name)
     except exceptions.NotFound:
         exists = False
     except:
         raise
     if not exists:
-        client.create_bucket(bucket_name)
+        conn.create_bucket(bucket_name)
 
 elif os.getenv('DATABASE_STORAGE') == "azure":
-    block_blob_service = BlockBlobService(account_name=os.getenv('WABS_ACCOUNT_NAME'), account_key=os.getenv('WABS_ACCESS_KEY'))
+    conn = BlobService(account_name=os.getenv('WABS_ACCOUNT_NAME'), account_key=os.getenv('WABS_ACCESS_KEY'))
     #It doesn't throw an exception if the container exists by default(https://github.com/Azure/azure-storage-python/blob/master/azure/storage/blob/baseblobservice.py#L504).
-    block_blob_service.create_container(bucket_name)
+    conn.create_container(bucket_name)
 
 elif os.getenv('DATABASE_STORAGE') == "swift":
     conn = swiftclient.Connection(
@@ -65,20 +59,15 @@ elif os.getenv('DATABASE_STORAGE') == "swift":
     conn.put_container(os.getenv('BUCKET_NAME'))
 
 else:
-    conn = boto3.resource('s3', endpoint_url=os.getenv('S3_URL'), config=Config(signature_version='s3v4'))
-    # stop boto3 from automatically changing the endpoint
-    conn.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
-    exists = True
-    try:
-        conn.meta.client.head_bucket(Bucket=bucket_name)
-    except botocore.exceptions.ClientError as e:
-        # If a client error is thrown, then check that it was a 404 error.
-        # If it was a 404 error, then the bucket does not exist.
-        error_code = int(e.response['Error']['Code'])
-        if error_code == 404:
-            exists = False
-        else:
-            raise
-
-    if not exists:
-        conn.create_bucket(Bucket=bucket_name)
+    botoconfig.add_section('s3')
+    botoconfig.set('s3', 'use-sigv4', 'True')
+    botoconfig.add_section('Boto')
+    botoconfig.set('Boto', 'is_secure', 'False')
+    conn = S3Connection(
+        host=os.getenv('S3_HOST'),
+        port=int(os.getenv('S3_PORT')),
+        calling_format=OrdinaryCallingFormat())
+    # HACK(bacongobbler): allow boto to connect to minio by changing the region name for s3v4 auth
+    conn.auth_region_name = os.getenv('AWS_REGION')
+    if not bucket_exists(conn, bucket_name):
+        conn.create_bucket(bucket_name)

rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh

@@ -6,25 +6,22 @@ if [[ "$DATABASE_STORAGE" == "s3" || "$DATABASE_STORAGE" == "minio" ]]; then
   AWS_ACCESS_KEY_ID=$(cat /var/run/secrets/deis/objectstore/creds/accesskey)
   AWS_SECRET_ACCESS_KEY=$(cat /var/run/secrets/deis/objectstore/creds/secretkey)
   if [[ "$DATABASE_STORAGE" == "s3" ]]; then
-    AWS_DEFAULT_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
+    AWS_REGION=$(cat /var/run/secrets/deis/objectstore/creds/region)
     BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-bucket)
   else
-    # these only need to be set if we're not accessing S3 (boto will figure this out)
-    echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > WALE_S3_ENDPOINT
-    if [ "$DEIS_MINIO_SERVICE_PORT" == "80" ]; then
-      # If you add port 80 to the end of the endpoint_url, boto3 freaks out.
-      # God I hate boto3 some days.
-      echo "http://$DEIS_MINIO_SERVICE_HOST" > S3_URL
-    else
-      echo "http://$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > S3_URL
-    fi
-    AWS_DEFAULT_REGION="us-east-1"
+    AWS_REGION="us-east-1"
     BUCKET_NAME="dbwal"
+    # these only need to be set if we're not accessing S3 (boto will figure this out)
+    echo "http+path://s3-$AWS_REGION.$DEIS_MINIO_SERVICE_HOST:$DEIS_MINIO_SERVICE_PORT" > WALE_S3_ENDPOINT
+    echo "$DEIS_MINIO_SERVICE_HOST" > S3_HOST
+    echo "$DEIS_MINIO_SERVICE_PORT" > S3_PORT
+    # enable sigv4 authentication
+    echo "true" > S3_USE_SIGV4
   fi
   echo "s3://$BUCKET_NAME" > WALE_S3_PREFIX
   echo $AWS_ACCESS_KEY_ID > AWS_ACCESS_KEY_ID
   echo $AWS_SECRET_ACCESS_KEY > AWS_SECRET_ACCESS_KEY
-  echo $AWS_DEFAULT_REGION > AWS_DEFAULT_REGION
+  echo $AWS_REGION > AWS_REGION
   echo $BUCKET_NAME > BUCKET_NAME
 elif [ "$DATABASE_STORAGE" == "gcs" ]; then
   GS_APPLICATION_CREDS="/var/run/secrets/deis/objectstore/creds/key.json"