|
1 | | -#!/usr/bin/env python3 |
2 | | - |
3 | | -import os |
4 | | - |
5 | | -import boto.s3 |
6 | | -import json |
7 | | -import swiftclient |
8 | | -from boto import config as botoconfig |
9 | | -from boto.exception import S3CreateError |
10 | | -from boto.s3.connection import S3Connection, OrdinaryCallingFormat |
11 | | -from oauth2client.service_account import ServiceAccountCredentials |
12 | | -from gcloud.storage.client import Client |
13 | | -from gcloud import exceptions |
14 | | -from azure.storage.blob import BlobService |
15 | | - |
16 | | -def bucket_exists(conn, name): |
17 | | - bucket = conn.lookup(name) |
18 | | - if not bucket: |
19 | | - return False |
20 | | - return True |
21 | | - |
22 | | -bucket_name = os.getenv('BUCKET_NAME') |
23 | | -region = os.getenv('S3_REGION') |
24 | | - |
25 | | -if os.getenv('DATABASE_STORAGE') == "s3": |
26 | | - conn = boto.s3.connect_to_region(region) |
27 | | - if not bucket_exists(conn, bucket_name): |
28 | | - try: |
29 | | - if region == "us-east-1": |
30 | | - # use "US Standard" region. workaround for https://github.com/boto/boto3/issues/125 |
31 | | - conn.create_bucket(bucket_name) |
32 | | - else: |
33 | | - conn.create_bucket(bucket_name, location=region) |
34 | | - # NOTE(bacongobbler): for versions prior to v2.9.0, the bucket is created in the default region. |
35 | | - # if we got here, we need to propagate "us-east-1" into WALE_S3_ENDPOINT because the bucket |
36 | | - # exists in a different region and we cannot find it. |
37 | | - # TODO(bacongobbler): deprecate this once we drop support for v2.8.0 and lower |
38 | | - except S3CreateError as err: |
39 | | - if region != 'us-east-1': |
40 | | - print('Failed to create bucket in {}. We are now assuming that the bucket was created in us-east-1.'.format(region)) |
41 | | - with open(os.path.join(os.environ['WALE_ENVDIR'], "WALE_S3_ENDPOINT"), "w+") as file: |
42 | | - file.write('https+path://s3.amazonaws.com:443') |
43 | | - else: |
44 | | - raise |
45 | | - |
46 | | -elif os.getenv('DATABASE_STORAGE') == "gcs": |
47 | | - scopes = ['https://www.googleapis.com/auth/devstorage.full_control'] |
48 | | - credentials = ServiceAccountCredentials.from_json_keyfile_name(os.getenv('GOOGLE_APPLICATION_CREDENTIALS'), scopes=scopes) |
49 | | - with open(os.getenv('GOOGLE_APPLICATION_CREDENTIALS')) as data_file: |
50 | | - data = json.load(data_file) |
51 | | - conn = Client(credentials=credentials, project=data['project_id']) |
52 | | - exists = True |
53 | | - try: |
54 | | - conn.get_bucket(bucket_name) |
55 | | - except exceptions.NotFound: |
56 | | - exists = False |
57 | | - except: |
58 | | - raise |
59 | | - if not exists: |
60 | | - conn.create_bucket(bucket_name) |
61 | | - |
62 | | -elif os.getenv('DATABASE_STORAGE') == "azure": |
63 | | - conn = BlobService(account_name=os.getenv('WABS_ACCOUNT_NAME'), account_key=os.getenv('WABS_ACCESS_KEY')) |
64 | | - #It doesn't throw an exception if the container exists by default(https://github.com/Azure/azure-storage-python/blob/master/azure/storage/blob/baseblobservice.py#L504). |
65 | | - conn.create_container(bucket_name) |
66 | | - |
67 | | -elif os.getenv('DATABASE_STORAGE') == "swift": |
68 | | - conn = swiftclient.Connection( |
69 | | - user=os.getenv('SWIFT_USER'), |
70 | | - key=os.getenv('SWIFT_PASSWORD'), |
71 | | - authurl=os.getenv('SWIFT_AUTHURL'), |
72 | | - auth_version=os.getenv('SWIFT_AUTH_VERSION'), |
73 | | - tenant_name=os.getenv('SWIFT_TENANT') |
74 | | - ) |
75 | | - # swift also does not throw exception if container already exists. |
76 | | - conn.put_container(os.getenv('BUCKET_NAME')) |
77 | | - |
78 | | -else: |
79 | | - if not botoconfig.has_section("s3"): |
80 | | - botoconfig.add_section('s3') |
81 | | - botoconfig.set('s3', 'use-sigv4', 'True') |
82 | | - if not botoconfig.has_section("Boto"): |
83 | | - botoconfig.add_section('Boto') |
84 | | - botoconfig.set('Boto', 'is_secure', 'False') |
85 | | - conn = S3Connection( |
86 | | - host=os.getenv('S3_HOST'), |
87 | | - port=int(os.getenv('S3_PORT')), |
88 | | - calling_format=OrdinaryCallingFormat()) |
89 | | - # HACK(bacongobbler): allow boto to connect to minio by changing the region name for s3v4 auth |
90 | | - conn.auth_region_name = os.getenv('S3_REGION') |
91 | | - if not bucket_exists(conn, bucket_name): |
92 | | - conn.create_bucket(bucket_name) |
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# shellcheck disable=SC1091 |
| 4 | +source /bin/normalize_storage |
| 5 | +has_bucket=$(mc ls minio -json|jq -r '.key'|grep -w "${MINIO_BUCKET}") |
| 6 | +if [ ! -n "$has_bucket" ] ;then |
| 7 | + mc mb minio/"${MINIO_BUCKET}" |
| 8 | +fi |
0 commit comments