Skip to content

Commit b66069b

Browse files
kmalakmala
authored
Merge pull request #125 from kmala/storage
feat(swift) : openstack swift support
2 parents f745e74 + bad5362 commit b66069b

7 files changed

Lines changed: 234 additions & 109 deletions

File tree

‎.travis.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ branches:
55
sudo: required
66
services:
77
- docker
8+
before_install:
9+
- sudo apt-get -qq update
10+
- sudo apt-get install -y python-swiftclient
811
env:
912
# HACK(bacongobbler): make travis tests work
1013
- DEIS_REGISTRY=travis-ci/

‎Makefile‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,12 @@ test-style:
3333
test-unit:
3434
@echo "Implement functional tests in _tests directory"
3535

36-
test-functional:
37-
contrib/ci/test.sh ${IMAGE}
36+
test-functional: test-functional-swift test-functional-minio
37+
38+
test-functional-minio:
39+
contrib/ci/test-minio.sh ${IMAGE}
40+
41+
test-functional-swift:
42+
contrib/ci/test-swift.sh ${IMAGE}
3843

3944
.PHONY: all docker-build docker-push test

‎contrib/ci/test-minio.sh‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#!/usr/bin/env bash
2+
3+
set -eof pipefail
4+
5+
TEST_ROOT=$(dirname "${BASH_SOURCE}")/
6+
source "${TEST_ROOT}/test.sh"
7+
8+
# make sure we are in this dir
9+
CURRENT_DIR=$(cd $(dirname $0); pwd)
10+
11+
create-postgres-creds
12+
13+
puts-step "creating fake minio credentials"
14+
15+
# create fake AWS credentials for minio admin credentials
16+
mkdir -p $CURRENT_DIR/tmp/aws-admin
17+
# needs to be 20 characters long
18+
echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-admin/access-key-id
19+
# needs to be 40 characters long
20+
echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-admin/access-secret-key
21+
22+
# create fake AWS credentials for minio user credentials
23+
mkdir -p $CURRENT_DIR/tmp/aws-user
24+
# needs to be 20 characters long
25+
echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-user/accesskey
26+
echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-user/access-key-id
27+
# needs to be 40 characters long
28+
echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-user/secretkey
29+
echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-user/access-secret-key
30+
31+
puts-step "creating fake kubernetes service account token"
32+
33+
# create fake k8s serviceaccount token for minio to "discover" itself
34+
mkdir -p $CURRENT_DIR/tmp/k8s
35+
echo "token" > $CURRENT_DIR/tmp/k8s/token
36+
echo "cert" > $CURRENT_DIR/tmp/k8s/ca.crt
37+
38+
# kill containers when this script exits or errors out
39+
trap 'kill-container $MINIO_JOB' INT TERM
40+
# boot minio
41+
MINIO_JOB=$(docker run -dv $CURRENT_DIR/tmp/aws-admin:/var/run/secrets/deis/minio/admin -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/minio/user -v $CURRENT_DIR/tmp/k8s:/var/run/secrets/kubernetes.io/serviceaccount quay.io/deisci/minio:canary boot server /home/minio/)
42+
43+
# boot postgres, linking the minio container and setting DEIS_MINIO_SERVICE_HOST and DEIS_MINIO_SERVICE_PORT
44+
PG_CMD="docker run -d --link $MINIO_JOB:minio -e BACKUP_FREQUENCY=1s -e DATABASE_STORAGE=minio -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/objectstore/creds $1"
45+
46+
# kill containers when this script exits or errors out
47+
trap 'kill-container $PG_JOB' INT TERM
48+
start-postgres "$PG_CMD"
49+
50+
# display logs for debugging purposes
51+
puts-step "displaying minio logs"
52+
docker logs $MINIO_JOB
53+
54+
check-postgres $PG_JOB
55+
56+
# check if minio has the 5 backups
57+
puts-step "checking if minio has 5 backups"
58+
BACKUPS="$(docker exec $MINIO_JOB ls /home/minio/dbwal/basebackups_005/ | grep json)"
59+
NUM_BACKUPS="$(docker exec $MINIO_JOB ls /home/minio/dbwal/basebackups_005/ | grep -c json)"
60+
# NOTE (bacongobbler): the BACKUP_FREQUENCY is only 1 second, so we could technically be checking
61+
# in the middle of a backup. Instead of failing, let's consider N+1 backups an acceptable case
62+
if [[ ! "$NUM_BACKUPS" -eq "5" && ! "$NUM_BACKUPS" -eq "6" ]]; then
63+
puts-error "did not find 5 or 6 base backups. 5 is the default, but 6 may exist if a backup is currently in progress (found $NUM_BACKUPS)"
64+
puts-error "$BACKUPS"
65+
exit 1
66+
fi
67+
68+
# kill off postgres, then reboot and see if it's running after recovering from backups
69+
puts-step "shutting off postgres, then rebooting to test data recovery"
70+
kill-container $PG_JOB
71+
72+
start-postgres "$PG_CMD"
73+
74+
check-postgres $PG_JOB

‎contrib/ci/test-swift.sh‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/usr/bin/env bash
2+
3+
set -eof pipefail
4+
5+
TEST_ROOT=$(dirname "${BASH_SOURCE}")/
6+
source "${TEST_ROOT}/test.sh"
7+
8+
# make sure we are in this dir
9+
CURRENT_DIR=$(cd $(dirname $0); pwd)
10+
11+
create-postgres-creds
12+
13+
puts-step "fetching openstack credentials"
14+
15+
# turn creds into something that we can use.
16+
mkdir -p $CURRENT_DIR/tmp/swift
17+
18+
# guess which value to use for tenant:
19+
TENANT=""
20+
21+
echo "test:tester" > $CURRENT_DIR/tmp/swift/username
22+
echo "testing" > $CURRENT_DIR/tmp/swift/password
23+
echo ${TENANT} > $CURRENT_DIR/tmp/swift/tenant
24+
echo "http://swift:8080/auth/v1.0" > $CURRENT_DIR/tmp/swift/authurl
25+
echo "1" > $CURRENT_DIR/tmp/swift/authversion
26+
echo "deis-swift-test" > $CURRENT_DIR/tmp/swift/database-container
27+
28+
# kill containers when this script exits or errors out
29+
trap 'kill-container $SWIFT_DATA' INT TERM
30+
# boot swift
31+
SWIFT_DATA=$(docker run -v /srv --name SWIFT_DATA busybox)
32+
33+
# kill containers when this script exits or errors out
34+
trap 'kill-container $SWIFT_JOB' INT TERM
35+
SWIFT_JOB=$(docker run --name onlyone --hostname onlyone -d -p 12345:8080 --volumes-from SWIFT_DATA -t deis/swift-onlyone:git-8516d23)
36+
37+
38+
# postgres container command
39+
PG_CMD="docker run -d --link $SWIFT_JOB:swift -e BACKUP_FREQUENCY=3s \
40+
-e DATABASE_STORAGE=swift \
41+
-v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds \
42+
-v $CURRENT_DIR/tmp/swift:/var/run/secrets/deis/objectstore/creds \
43+
$1"
44+
45+
# kill containers when this script exits or errors out
46+
trap 'kill-container $PG_JOB' INT TERM
47+
start-postgres "$PG_CMD"
48+
49+
# display logs for debugging purposes
50+
puts-step "displaying swift logs"
51+
docker logs $SWIFT_JOB
52+
53+
check-postgres $PG_JOB
54+
55+
# check if swift has some backups ... 3 ?
56+
puts-step "checking if swift has at least 3 backups"
57+
58+
BACKUPS="$(swift -A http://127.0.0.1:12345/auth/v1.0 -U test:tester -K testing list deis-swift-test | grep basebackups_005 | grep json)"
59+
NUM_BACKUPS="$(swift -A http://127.0.0.1:12345/auth/v1.0 -U test:tester -K testing list deis-swift-test | grep basebackups_005 | grep -c json)"
60+
# NOTE (bacongobbler): the BACKUP_FREQUENCY is only 1 second, so we could technically be checking
61+
# in the middle of a backup. Instead of failing, let's consider N+1 backups an acceptable case
62+
if [[ ! "$NUM_BACKUPS" -eq "5" && ! "$NUM_BACKUPS" -eq "6" ]]; then
63+
puts-error "did not find 5 or 6 base backups. 5 is the default, but 6 may exist if a backup is currently in progress (found $NUM_BACKUPS)"
64+
puts-error "$BACKUPS"
65+
exit 1
66+
fi
67+
68+
# kill off postgres, then reboot and see if it's running after recovering from backups
69+
puts-step "shutting off postgres, then rebooting to test data recovery"
70+
kill-container $PG_JOB
71+
72+
start-postgres "$PG_CMD"
73+
74+
check-postgres $PG_JOB
75+
76+
puts-step "tests PASSED!"
77+
exit 0

‎contrib/ci/test.sh‎

Lines changed: 24 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -10,93 +10,33 @@ puts-error() {
1010
echo "!!! $@"
1111
}
1212

13-
kill-containers() {
14-
puts-step "destroying containers"
15-
docker rm -f "$MINIO_JOB" "$PG_JOB"
13+
kill-container() {
14+
puts-step "destroying container $1"
15+
docker rm -f "$1"
1616
}
1717

18-
# make sure we are in this dir
19-
CURRENT_DIR=$(cd $(dirname $0); pwd)
18+
create-postgres-creds() {
19+
puts-step "creating fake postgres credentials"
2020

21-
puts-step "creating fake postgres credentials"
22-
23-
# create fake postgres credentials
24-
mkdir -p $CURRENT_DIR/tmp/creds
25-
echo "testuser" > $CURRENT_DIR/tmp/creds/user
26-
echo "icanttellyou" > $CURRENT_DIR/tmp/creds/password
27-
28-
puts-step "creating fake minio credentials"
29-
30-
# create fake AWS credentials for minio admin credentials
31-
mkdir -p $CURRENT_DIR/tmp/aws-admin
32-
# needs to be 20 characters long
33-
echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-admin/access-key-id
34-
# needs to be 40 characters long
35-
echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-admin/access-secret-key
36-
37-
# create fake AWS credentials for minio user credentials
38-
mkdir -p $CURRENT_DIR/tmp/aws-user
39-
# needs to be 20 characters long
40-
echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-user/accesskey
41-
echo "12345678901234567890" > $CURRENT_DIR/tmp/aws-user/access-key-id
42-
# needs to be 40 characters long
43-
echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-user/secretkey
44-
echo "1234567890123456789012345678901234567890" > $CURRENT_DIR/tmp/aws-user/access-secret-key
45-
46-
puts-step "creating fake kubernetes service account token"
47-
48-
# create fake k8s serviceaccount token for minio to "discover" itself
49-
mkdir -p $CURRENT_DIR/tmp/k8s
50-
echo "token" > $CURRENT_DIR/tmp/k8s/token
51-
echo "cert" > $CURRENT_DIR/tmp/k8s/ca.crt
52-
53-
# boot minio
54-
MINIO_JOB=$(docker run -dv $CURRENT_DIR/tmp/aws-admin:/var/run/secrets/deis/minio/admin -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/minio/user -v $CURRENT_DIR/tmp/k8s:/var/run/secrets/kubernetes.io/serviceaccount quay.io/deisci/minio:canary boot server /home/minio/)
55-
56-
# boot postgres, linking the minio container and setting DEIS_MINIO_SERVICE_HOST and DEIS_MINIO_SERVICE_PORT
57-
PG_JOB=$(docker run -d --link $MINIO_JOB:minio -e BACKUP_FREQUENCY=1s -e DATABASE_STORAGE=minio -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/objectstore/creds $1)
58-
59-
# kill containers when this script exits or errors out
60-
trap kill-containers INT TERM
61-
62-
# wait for postgres to boot
63-
puts-step "sleeping for 90s while postgres is booting..."
64-
sleep 90s
65-
66-
# display logs for debugging purposes
67-
puts-step "displaying minio logs"
68-
docker logs $MINIO_JOB
69-
puts-step "displaying postgres logs"
70-
docker logs $PG_JOB
71-
72-
# check if postgres is running
73-
puts-step "checking if postgres is running"
74-
docker exec $PG_JOB is_running
75-
76-
# check if minio has the 5 backups
77-
puts-step "checking if minio has 5 backups"
78-
BACKUPS="$(docker exec $MINIO_JOB ls /home/minio/dbwal/basebackups_005/ | grep json)"
79-
NUM_BACKUPS="$(docker exec $MINIO_JOB ls /home/minio/dbwal/basebackups_005/ | grep -c json)"
80-
# NOTE (bacongobbler): the BACKUP_FREQUENCY is only 1 second, so we could technically be checking
81-
# in the middle of a backup. Instead of failing, let's consider N+1 backups an acceptable case
82-
if [[ ! "$NUM_BACKUPS" -eq "5" && ! "$NUM_BACKUPS" -eq "6" ]]; then
83-
puts-error "did not find 5 or 6 base backups. 5 is the default, but 6 may exist if a backup is currently in progress (found $NUM_BACKUPS)"
84-
puts-error "$BACKUPS"
85-
exit 1
86-
fi
87-
88-
# kill off postgres, then reboot and see if it's running after recovering from backups
89-
puts-step "shutting off postgres, then rebooting to test data recovery"
90-
docker rm -f $PG_JOB
91-
PG_JOB=$(docker run -d --link $MINIO_JOB:minio -e BACKUP_FREQUENCY=1s -e DATABASE_STORAGE=minio -e DEIS_MINIO_SERVICE_HOST=minio -e DEIS_MINIO_SERVICE_PORT=9000 -v $CURRENT_DIR/tmp/creds:/var/run/secrets/deis/database/creds -v $CURRENT_DIR/tmp/aws-user:/var/run/secrets/deis/objectstore/creds $1)
21+
# create fake postgres credentials
22+
mkdir -p $CURRENT_DIR/tmp/creds
23+
echo "testuser" > $CURRENT_DIR/tmp/creds/user
24+
echo "icanttellyou" > $CURRENT_DIR/tmp/creds/password
25+
}
9226

93-
# wait for postgres to boot
94-
puts-step "sleeping for 90s while postgres is recovering from backup..."
95-
sleep 90s
27+
start-postgres() {
28+
PG_JOB=$($1)
29+
# wait for postgres to boot
30+
puts-step "sleeping for 90s while postgres is booting..."
31+
sleep 90s
32+
}
9633

97-
puts-step "displaying postgres logs"
98-
docker logs $PG_JOB
34+
check-postgres() {
35+
# display logs for debugging purposes
36+
puts-step "displaying postgres logs"
37+
docker logs $1
9938

100-
# check if postgres is running
101-
puts-step "checking if postgres is running"
102-
docker exec $PG_JOB is_running
39+
# check if postgres is running
40+
puts-step "checking if postgres is running"
41+
docker exec $1 is_running
42+
}

‎rootfs/bin/create_bucket‎

Lines changed: 34 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
#!/usr/bin/env python
22

33
import os
4-
import sys
54

65
import boto3
76
import botocore
87
import json
8+
import swiftclient
99
from botocore.utils import fix_s3_host
1010
from botocore.client import Config
1111
from oauth2client.service_account import ServiceAccountCredentials
@@ -30,7 +30,7 @@ if os.getenv('DATABASE_STORAGE') == "s3":
3030
raise
3131

3232
if not exists:
33-
conn.create_bucket(Bucket=bucket_name)
33+
conn.create_bucket(Bucket=bucket_name)
3434

3535
elif os.getenv('DATABASE_STORAGE') == "gcs":
3636
scopes = ['https://www.googleapis.com/auth/devstorage.full_control']
@@ -46,28 +46,39 @@ elif os.getenv('DATABASE_STORAGE') == "gcs":
4646
except:
4747
raise
4848
if not exists:
49-
client.create_bucket(bucket_name)
49+
client.create_bucket(bucket_name)
5050

5151
elif os.getenv('DATABASE_STORAGE') == "azure":
52-
block_blob_service = BlockBlobService(account_name=os.getenv('WABS_ACCOUNT_NAME'), account_key=os.getenv('WABS_ACCESS_KEY'))
53-
#It doesn't throw an exception if the container exists by default(https://github.com/Azure/azure-storage-python/blob/master/azure/storage/blob/baseblobservice.py#L504).
54-
block_blob_service.create_container(bucket_name)
52+
block_blob_service = BlockBlobService(account_name=os.getenv('WABS_ACCOUNT_NAME'), account_key=os.getenv('WABS_ACCESS_KEY'))
53+
#It doesn't throw an exception if the container exists by default(https://github.com/Azure/azure-storage-python/blob/master/azure/storage/blob/baseblobservice.py#L504).
54+
block_blob_service.create_container(bucket_name)
5555

56-
else :
57-
conn = boto3.resource('s3', endpoint_url=os.getenv('S3_URL'), config=Config(signature_version='s3v4'))
58-
# stop boto3 from automatically changing the endpoint
59-
conn.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
60-
exists = True
61-
try:
62-
conn.meta.client.head_bucket(Bucket=bucket_name)
63-
except botocore.exceptions.ClientError as e:
64-
# If a client error is thrown, then check that it was a 404 error.
65-
# If it was a 404 error, then the bucket does not exist.
66-
error_code = int(e.response['Error']['Code'])
67-
if error_code == 404:
68-
exists = False
69-
else:
70-
raise
56+
elif os.getenv('DATABASE_STORAGE') == "swift":
57+
conn = swiftclient.Connection(
58+
user=os.getenv('SWIFT_USER'),
59+
key=os.getenv('SWIFT_PASSWORD'),
60+
authurl=os.getenv('SWIFT_AUTHURL'),
61+
auth_version=os.getenv('SWIFT_AUTH_VERSION'),
62+
tenant_name=os.getenv('SWIFT_TENANT')
63+
)
64+
# swift also does not throw exception if container already exists.
65+
conn.put_container(os.getenv('BUCKET_NAME'))
7166

72-
if not exists:
73-
conn.create_bucket(Bucket=bucket_name)
67+
else:
68+
conn = boto3.resource('s3', endpoint_url=os.getenv('S3_URL'), config=Config(signature_version='s3v4'))
69+
# stop boto3 from automatically changing the endpoint
70+
conn.meta.client.meta.events.unregister('before-sign.s3', fix_s3_host)
71+
exists = True
72+
try:
73+
conn.meta.client.head_bucket(Bucket=bucket_name)
74+
except botocore.exceptions.ClientError as e:
75+
# If a client error is thrown, then check that it was a 404 error.
76+
# If it was a 404 error, then the bucket does not exist.
77+
error_code = int(e.response['Error']['Code'])
78+
if error_code == 404:
79+
exists = False
80+
else:
81+
raise
82+
83+
if not exists:
84+
conn.create_bucket(Bucket=bucket_name)

‎rootfs/docker-entrypoint-initdb.d/001_setup_envdir.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,4 +40,19 @@ elif [ "$DATABASE_STORAGE" == "azure" ]; then
4040
echo $WABS_ACCESS_KEY > WABS_ACCESS_KEY
4141
echo "wabs://$BUCKET_NAME" > WALE_WABS_PREFIX
4242
echo $BUCKET_NAME > BUCKET_NAME
43+
elif [ "$DATABASE_STORAGE" == "swift" ]; then
44+
SWIFT_USER=$(cat /var/run/secrets/deis/objectstore/creds/username)
45+
SWIFT_PASSWORD=$(cat /var/run/secrets/deis/objectstore/creds/password)
46+
SWIFT_TENANT=$(cat /var/run/secrets/deis/objectstore/creds/tenant)
47+
SWIFT_AUTHURL=$(cat /var/run/secrets/deis/objectstore/creds/authurl)
48+
SWIFT_AUTH_VERSION=$(cat /var/run/secrets/deis/objectstore/creds/authversion)
49+
BUCKET_NAME=$(cat /var/run/secrets/deis/objectstore/creds/database-container)
50+
# set defaults for variables that we can guess at
51+
echo $SWIFT_USER > SWIFT_USER
52+
echo $SWIFT_PASSWORD > SWIFT_PASSWORD
53+
echo $SWIFT_TENANT > SWIFT_TENANT
54+
echo $SWIFT_AUTHURL > SWIFT_AUTHURL
55+
echo $SWIFT_AUTH_VERSION > SWIFT_AUTH_VERSION
56+
echo "swift://$BUCKET_NAME" > WALE_SWIFT_PREFIX
57+
echo $BUCKET_NAME > BUCKET_NAME
4358
fi

0 commit comments

Comments
 (0)