Skip to content

Commit b7c2990

Browse files
carmstrongcarmstrong
committed
feat(database): use deis-store and WAL-e to ship WAL logs
feat(database): perform backups periodically feat(database): perform initial backup after db creation fix(database): perform base backup after database is running feat(database): make startup more verbose fix(database): correctly determine if database is initialized fix(database): properly escape recovery.conf fix(database): don't start backing up if a restore is in progress ref(database): template deis-store-gateway endpoint and port fix(database): database is already created on package installation ref(database): no longer use custom confd fix(database): fix functional test check for output ref(database): remove volume and checkout specific wal-e SHA fix(database): fix perms to match new trusty images fix(database): dont remove container on start-pre fix(database): report running after initial backup
1 parent b5cb742 commit b7c2990

15 files changed

Lines changed: 157 additions & 21 deletions

‎database/Dockerfile‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,37 @@ RUN curl -sSL -o /usr/local/bin/etcdctl https://s3-us-west-2.amazonaws.com/opdem
1313
RUN curl -sSL -o /usr/local/bin/confd https://s3-us-west-2.amazonaws.com/opdemand/confd-v0.5.0-json \
1414
&& chmod +x /usr/local/bin/confd
1515

16-
# install 9.3 from postgresql.org repository
16+
# install postgresql 9.3 from postgresql.org repository as well as requirements for building wal-e
1717
RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" > /etc/apt/sources.list.d/pgdg.list
1818
RUN curl -sk https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
19-
RUN apt-get update && apt-get install -yq postgresql-9.3 && /etc/init.d/postgresql stop
19+
RUN apt-get update && apt-get install -yq \
20+
curl \
21+
daemontools \
22+
file \
23+
gcc \
24+
git \
25+
libxml2-dev \
26+
libxslt1-dev \
27+
lzop \
28+
postgresql-9.3 \
29+
pv \
30+
python-dev
31+
32+
RUN /etc/init.d/postgresql stop
33+
34+
# install pip
35+
RUN curl -sSL https://raw.githubusercontent.com/pypa/pip/1.5.6/contrib/get-pip.py | python -
36+
37+
# install wal-e
38+
WORKDIR /tmp
39+
RUN git clone https://github.com/wal-e/wal-e.git
40+
WORKDIR /tmp/wal-e
41+
RUN git checkout c6f0d02
42+
RUN pip install .
43+
RUN mkdir -p /etc/wal-e.d/env
44+
RUN chown -R root:postgres /etc/wal-e.d
2045

2146
# define the execution environment
22-
VOLUME ["/var/lib/postgresql"]
2347
WORKDIR /app
2448
CMD ["/app/bin/boot"]
2549
EXPOSE 5432

‎database/bin/boot‎

Lines changed: 56 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,15 @@ export ETCD="$HOST:$ETCD_PORT"
1515
export ETCD_PATH=${ETCD_PATH:-/deis/database}
1616
export ETCD_TTL=${ETCD_TTL:-10}
1717

18-
# initialize data volume
19-
if [[ ! -d /var/lib/postgresql/9.3/main ]]; then
20-
mkdir -p /var/lib/postgresql/9.3
21-
chown -R postgres:postgres /var/lib/postgresql
22-
sudo -u postgres /usr/lib/postgresql/9.3/bin/initdb -D /var/lib/postgresql/9.3/main
23-
else
24-
chown -R postgres:postgres /var/lib/postgresql
25-
fi
18+
export BUCKET_NAME=${BUCKET_NAME:-db_wal}
19+
export BACKUPS_TO_RETAIN=${BACKUPS_TO_RETAIN:-5}
20+
21+
# how many TTL/2 sleeps between backups -- 2160 is 3 hours
22+
export BACKUP_FREQUENCY=${BACKUP_FREQUENCY:-2160}
2623

2724
# wait for etcd to be available
2825
until etcdctl --no-sync -C $ETCD ls >/dev/null 2>&1; do
29-
echo "waiting for etcd at $ETCD..."
26+
echo "database: waiting for etcd at $ETCD..."
3027
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
3128
done
3229

@@ -43,13 +40,39 @@ etcd_set_default adminPass ${PG_ADMIN_PASS:-changeme123}
4340
etcd_set_default user ${PG_USER_NAME:-deis}
4441
etcd_set_default password ${PG_USER_PASS:-changeme123}
4542
etcd_set_default name ${PG_USER_DB:-deis}
43+
etcd_set_default bucketName ${BUCKET_NAME}
4644

4745
# wait for confd to run once and install initial templates
4846
until confd -onetime -node $ETCD -config-file /app/confd.toml; do
4947
echo "database: waiting for confd to write initial templates..."
5048
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
5149
done
5250

51+
# ensure WAL log bucket exists
52+
envdir /etc/wal-e.d/env /app/bin/create_bucket ${BUCKET_NAME}
53+
54+
initial_backup=0
55+
if [[ ! -f /var/lib/postgresql/9.3/main/initialized ]]; then
56+
echo "database: no existing database found."
57+
# check if there are any backups -- if so, let's restore
58+
# we could probably do better than just testing number of lines -- one line is just a heading, meaning no backups
59+
if [[ `envdir /etc/wal-e.d/env wal-e --terse backup-list | wc -l` -gt "1" ]]; then
60+
echo "database: restoring from backup..."
61+
rm -rf /var/lib/postgresql/9.3/main
62+
sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-fetch /var/lib/postgresql/9.3/main LATEST
63+
chown -R postgres:postgres /var/lib/postgresql/9.3/main
64+
chmod 0700 /var/lib/postgresql/9.3/main
65+
echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" | sudo -u postgres tee /var/lib/postgresql/9.3/main/recovery.conf >/dev/null
66+
else
67+
echo "database: no backups found. Initializing a new database..."
68+
initial_backup=1
69+
fi
70+
# either way, we mark the database as initialized
71+
touch /var/lib/postgresql/9.3/main/initialized
72+
else
73+
echo "database: existing data directory found. Starting postgres..."
74+
fi
75+
5376
# run the service in the background
5477
sudo -i -u postgres /usr/lib/postgresql/9.3/bin/postgres \
5578
-c config-file=${PG_CONFIG:-/etc/postgresql/9.3/main/postgresql.conf} \
@@ -75,11 +98,17 @@ sleep 1 && while [[ -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".5432\
7598
# perform a one-time reload to populate database entries
7699
/usr/local/bin/reload
77100

78-
echo deis-database running...
101+
if [[ "${initial_backup}" == "1" ]] ; then
102+
echo "database: performing an initial backup..."
103+
# perform an initial backup
104+
sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
105+
fi
106+
107+
echo "database: postgres is running..."
79108

109+
count=1
80110
# publish the service to etcd using the injected HOST and EXTERNAL_PORT
81111
if [[ ! -z $EXTERNAL_PORT ]]; then
82-
83112
# configure service discovery
84113
PORT=${PORT:-5432}
85114
PROTO=${PROTO:-tcp}
@@ -93,6 +122,22 @@ if [[ ! -z $EXTERNAL_PORT ]]; then
93122
while [[ ! -z $(netstat -lnt | awk "\$6 == \"LISTEN\" && \$4 ~ \".$PORT\" && \$1 ~ \"$PROTO.?\"") ]] ; do
94123
etcdctl --no-sync -C $ETCD set $ETCD_PATH/host $HOST --ttl $ETCD_TTL >/dev/null
95124
etcdctl --no-sync -C $ETCD set $ETCD_PATH/port $EXTERNAL_PORT --ttl $ETCD_TTL >/dev/null
125+
126+
# perform a backup whenever we hit BACKUP_FREQUENCY
127+
# we really need cron :(
128+
if [[ "${count}" -ge "${BACKUP_FREQUENCY}" ]] ; then
129+
echo "database: performing a backup..."
130+
if [[ -f /var/lib/postgresql/9.3/main/recovery.conf ]] ; then
131+
echo "database: database is currently recovering from a backup. Will try again next loop..."
132+
else
133+
# perform a backup
134+
sudo -u postgres envdir /etc/wal-e.d/env wal-e backup-push /var/lib/postgresql/9.3/main
135+
# only retain the latest BACKUPS_TO_RETAIN backups
136+
sudo -u postgres envdir /etc/wal-e.d/env wal-e delete --confirm retain ${BACKUPS_TO_RETAIN}
137+
count=0
138+
fi
139+
fi
140+
((count++))
96141
sleep $(($ETCD_TTL/2)) # sleep for half the TTL
97142
done
98143

‎database/conf.d/AWS_ACCESS_KEY_ID.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[template]
2+
src = "AWS_ACCESS_KEY_ID"
3+
dest = "/etc/wal-e.d/env/AWS_ACCESS_KEY_ID"
4+
uid = 0
5+
gid = 106
6+
mode = "0640"
7+
keys = [
8+
"/deis/store/gateway",
9+
]
10+
check_cmd = "/app/bin/check {{ .src }}"

‎database/conf.d/AWS_SECRET_ACCESS_KEY.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[template]
2+
src = "AWS_SECRET_ACCESS_KEY"
3+
dest = "/etc/wal-e.d/env/AWS_SECRET_ACCESS_KEY"
4+
uid = 0
5+
gid = 106
6+
mode = "0640"
7+
keys = [
8+
"/deis/store/gateway",
9+
]
10+
check_cmd = "/app/bin/check {{ .src }}"

‎database/conf.d/WALE_S3_ENDPOINT.toml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
[template]
2+
src = "WALE_S3_ENDPOINT"
3+
dest = "/etc/wal-e.d/env/WALE_S3_ENDPOINT"
4+
uid = 0
5+
gid = 106
6+
mode = "0640"
7+
keys = [
8+
"/deis/store/gateway",
9+
]
10+
check_cmd = "/app/bin/check {{ .src }}"

‎database/conf.d/WALE_S3_PREFIX.toml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[template]
2+
src = "WALE_S3_PREFIX"
3+
dest = "/etc/wal-e.d/env/WALE_S3_PREFIX"
4+
uid = 0
5+
gid = 106
6+
mode = "0640"
7+
keys = [
8+
"/deis/store/gateway",
9+
"/deis/database",
10+
]
11+
check_cmd = "/app/bin/check {{ .src }}"

‎database/conf.d/create_bucket.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[template]
2+
src = "create_bucket"
3+
dest = "/app/bin/create_bucket"
4+
uid = 0
5+
gid = 0
6+
mode = "0755"
7+
keys = [
8+
"/deis/store/gateway",
9+
]

‎database/templates/AWS_ACCESS_KEY_ID‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{{ .deis_store_gateway_accessKey }}

‎database/templates/AWS_SECRET_ACCESS_KEY‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{{ .deis_store_gateway_secretKey }}

‎database/templates/WALE_S3_ENDPOINT‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
http+path://{{ .deis_store_gateway_host }}:{{ .deis_store_gateway_port }}

0 commit comments

Comments
 (0)