Skip to content

Commit 8477f3e

Browse files
aledbfmboersma
aledbf
authored and
mboersma
committed
feat(store): add gateway mock
1 parent 18cf1a9 commit 8477f3e

5 files changed

Lines changed: 168 additions & 0 deletions

File tree

‎store/Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ GATEWAY_IMAGE = $(IMAGE_PREFIX)store-gateway:$(BUILD_TAG)
99
GATEWAY_DEV_IMAGE = $(DEV_REGISTRY)/$(GATEWAY_IMAGE)
1010
METADATA_IMAGE = $(IMAGE_PREFIX)store-metadata:$(BUILD_TAG)
1111
METADATA_DEV_IMAGE = $(DEV_REGISTRY)/$(METADATA_IMAGE)
12+
MOCK_IMAGE = $(IMAGE_PREFIX)store-mock:$(BUILD_TAG)
13+
MOCK_DEV_IMAGE = $(DEV_REGISTRY)/$(MOCK_IMAGE)
1214
MONITOR_IMAGE = $(IMAGE_PREFIX)store-monitor:$(BUILD_TAG)
1315
MONITOR_DEV_IMAGE = $(DEV_REGISTRY)/$(MONITOR_IMAGE)
1416

@@ -20,6 +22,7 @@ build: check-docker
2022
docker build -t deis/store-$(I):$(BUILD_TAG) $(I)/ ; \
2123
rm $(I)/Dockerfile ; \
2224
)
25+
docker build -t $(MOCK_IMAGE) mock/
2326

2427
clean: check-docker check-registry
2528
$(foreach I, $(BUILT_IMAGES), \
@@ -74,6 +77,8 @@ push: check-registry
7477
docker push $(GATEWAY_DEV_IMAGE)
7578
docker tag $(METADATA_IMAGE) $(METADATA_DEV_IMAGE)
7679
docker push $(METADATA_DEV_IMAGE)
80+
docker tag $(MOCK_IMAGE) $(MOCK_DEV_IMAGE)
81+
docker push $(MOCK_DEV_IMAGE)
7782
docker tag $(MONITOR_IMAGE) $(MONITOR_DEV_IMAGE)
7883
docker push $(MONITOR_DEV_IMAGE)
7984

@@ -87,6 +92,7 @@ release:
8792
docker push $(DAEMON_IMAGE)
8893
docker push $(GATEWAY_IMAGE)
8994
docker push $(METADATA_IMAGE)
95+
docker push $(MOCK_IMAGE)
9096
docker push $(MONITOR_IMAGE)
9197

9298
deploy: build dev-release restart

‎store/mock/Dockerfile‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
FROM ubuntu-debootstrap:14.04
2+
3+
ENV DEBIAN_FRONTEND noninteractive
4+
5+
WORKDIR /app
6+
EXPOSE 8888
7+
CMD ["/app/bin/boot"]
8+
ADD bin/boot /app/bin/boot
9+
10+
ADD build.sh /tmp/build.sh
11+
RUN DOCKER_BUILD=true /tmp/build.sh

‎store/mock/README.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Mock S3 storage
2+
3+
The objective is to not require a Ceph cluster in the test to provide a S3 compatible service.
4+
This component uses [mock-s3](https://github.com/jserver/mock-s3).
5+
6+
## Usage:
7+
8+
```
9+
docker run -p 8888:8888 -e HOST=$COREOS_PRIVATE_IPV4 -v <local directory>:/app/storage deis/store-mock
10+
```
11+
12+
*The use of a local directory `(-v <local directory>)` is optional*
13+
14+
15+
`mock-s3` does not requires an ACCESS_KEY and SECRET_KEY (there is no concept of permissions) but this
16+
component will generate both keep compatibility with `deis-store-gateway`
17+
18+
## Containers
19+
20+
The mock store component is composed by one container:
21+
22+
* [store-mock](https://index.docker.io/u/deis/store-mock/) - the blob store gateway,
23+
offering a S3-compatible bucket APIs using the local filesystem as storage.
24+
25+
## Usage
26+
27+
Please consult the [Makefile](Makefile) for current instructions on how to build, test, push,
28+
install, and start **deis/store-mock**.
29+
30+
## License
31+
32+
© 2014 OpDemand LLC
33+
34+
Licensed under the Apache License, Version 2.0 (the "License"); you may
35+
not use this file except in compliance with the License. You may obtain
36+
a copy of the License at <http://www.apache.org/licenses/LICENSE-2.0>
37+
38+
Unless required by applicable law or agreed to in writing, software
39+
distributed under the License is distributed on an "AS IS" BASIS,
40+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
41+
See the License for the specific language governing permissions and
42+
limitations under the License.

‎store/mock/bin/boot‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
#!/bin/bash
2+
#
3+
# This script is designed to be run inside the container
4+
#
5+
6+
# fail hard and fast even on pipelines
7+
set -eo pipefail
8+
9+
# set debug based on envvar
10+
[[ $DEBUG ]] && set -x
11+
12+
# configure etcd
13+
export ETCD_PORT=${ETCD_PORT:-4001}
14+
export ETCD="$HOST:$ETCD_PORT"
15+
export ETCD_PATH=${ETCD_PATH:-/deis/store/gateway}
16+
export ETCD_TTL=${ETCD_TTL:-10}
17+
18+
export EXTERNAL_PORT=${EXTERNAL_PORT:-8888}
19+
export STORAGE_DIRECTORY=${STORAGE_DIRECTORY:-/app/storage}
20+
21+
# wait for etcd to be available
22+
until etcdctl --no-sync -C $ETCD ls >/dev/null 2>&1; do
23+
echo "waiting for etcd at $ETCD..."
24+
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
25+
done
26+
27+
# wait until etcd has discarded potentially stale values
28+
sleep $(($ETCD_TTL+1))
29+
30+
function etcd_safe_mkdir {
31+
set +e
32+
etcdctl --no-sync -C $ETCD mkdir $1 >/dev/null 2>&1
33+
if [[ $? -ne 0 && $? -ne 4 ]]; then
34+
echo "etcd_safe_mkdir: an etcd error occurred. aborting..."
35+
exit 1
36+
fi
37+
set -e
38+
}
39+
40+
etcd_safe_mkdir $ETCD_PATH
41+
42+
# store the access key and secret key for consumption by other services
43+
ACCESS_KEY=`etcdctl --no-sync -C $ETCD get $ETCD_PATH/accessKey 2>/dev/null || openssl rand -base64 21 | tr -d '\n'`
44+
SECRET_KEY=`etcdctl --no-sync -C $ETCD get $ETCD_PATH/secretKey 2>/dev/null || openssl rand -base64 64 | tr -d '\n'`
45+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/accessKey ${ACCESS_KEY} >/dev/null
46+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/secretKey ${SECRET_KEY} >/dev/null
47+
48+
mkdir -p /app/storage
49+
50+
mock_s3 --hostname 0.0.0.0 --port $EXTERNAL_PORT --root $STORAGE_DIRECTORY &
51+
52+
echo deis-store-gateway running...
53+
54+
# configure service discovery
55+
PROTO=${PROTO:-tcp}
56+
57+
set +e
58+
59+
# wait for the service to become available on PUBLISH port
60+
sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do sleep 1; done
61+
62+
# while the port is listening, publish to etcd
63+
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PUBLISH\" && \$1 ~ \"$PROTO.?\"") ]] ; do
64+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
65+
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $EXTERNAL_PORT --ttl $ETCD_TTL >/dev/null
66+
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
67+
done
68+
69+
# if the loop quits, something went wrong
70+
exit 1

‎store/mock/build.sh‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
# fail on any command exiting non-zero
4+
set -eo pipefail
5+
6+
if [[ -z $DOCKER_BUILD ]]; then
7+
echo
8+
echo "Note: this script is intended for use by the Dockerfile and not as a way to build the store mock component locally"
9+
echo
10+
exit 1
11+
fi
12+
13+
# install required packages to build
14+
apt-get update \
15+
&& apt-get install -y build-essential git python-dev curl net-tools
16+
17+
# install etcdctl
18+
curl -sSL -o /usr/local/bin/etcdctl https://s3-us-west-2.amazonaws.com/opdemand/etcdctl-v0.4.6 \
19+
&& chmod +x /usr/local/bin/etcdctl
20+
21+
22+
git clone https://github.com/jserver/mock-s3 /app/mock-s3
23+
24+
cd /app/mock-s3
25+
26+
# install pip
27+
curl -sSL https://raw.githubusercontent.com/pypa/pip/1.5.6/contrib/get-pip.py | python -
28+
29+
python setup.py install
30+
31+
# cleanup. indicate that python, libpq and libyanl are required packages.
32+
apt-mark unmarkauto python && \
33+
apt-get remove -y --purge build-essential python-dev gcc cpp git && \
34+
apt-get autoremove -y --purge && \
35+
apt-get clean -y && \
36+
rm -Rf /usr/share/man /usr/share/doc && \
37+
rm -rf /tmp/* /var/tmp/* && \
38+
rm -rf /var/lib/apt/lists/*
39+

0 commit comments

Comments
 (0)