Skip to content

Commit 6b452e5

Browse files
committed
fix(services): prevent units from stopping as "failed/failed"
Deis' systemd units always return "failed/failed" after being stopped by fleetctl. Somehow our ExecStop command to remove the service container conflicts with our ExecStart command and causes it to return an error code. This is fixed by removing the ExecStop commands and adding a `--rm` flag to the docker command in ExecStart. Additionally, our on_exit() handler in each /app/bin/boot script was not returning 0. TESTING: Run `make run`, then `make stop` on a Deis cluster. You should see all units stop and return to "inactive/dead" status, whereas before they would show "failed/failed."
1 parent 813057e commit 6b452e5

14 files changed

Lines changed: 16 additions & 16 deletions

File tree

builder/bin/boot

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ SSHD_PID=$!
6565
function on_exit() {
6666
kill -TERM $DOCKER_PID $SSHD_PID
6767
wait $DOCKER_PID $SSHD_PID 2>/dev/null
68+
exit 0
6869
}
6970
trap on_exit INT TERM EXIT
7071

builder/systemd/deis-builder.service

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@ TimeoutStartSec=20m
77
ExecStartPre=/bin/sh -c "docker history deis/builder >/dev/null || docker pull deis/builder:latest"
88
ExecStartPre=/bin/sh -c "docker inspect deis-builder >/dev/null && docker rm -f deis-builder || true"
99
ExecStartPre=/bin/sh -c "docker inspect deis-builder-data >/dev/null 2>&1 || docker run --name deis-builder-data -v /var/lib/docker deis/base true"
10-
ExecStart=/usr/bin/docker run --name deis-builder -p 2223:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2223 --volumes-from deis-builder-data --privileged deis/builder
10+
ExecStart=/usr/bin/docker run --name deis-builder --rm -p 2223:22 -e PUBLISH=22 -e HOST=${COREOS_PRIVATE_IPV4} -e PORT=2223 --volumes-from deis-builder-data --privileged deis/builder
1111
ExecStartPost=/bin/sh -c "echo 'Waiting for builder on 2223/tcp...' && until cat </dev/null>/dev/tcp/$COREOS_PRIVATE_IPV4/2223; do sleep 1; done"
12-
ExecStop=/usr/bin/docker rm -f deis-builder
1312

1413
[Install]
1514
WantedBy=multi-user.target

cache/bin/boot

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ SERVICE_PID=$!
4444
function on_exit() {
4545
kill -TERM $SERVICE_PID
4646
wait $SERVICE_PID 2>/dev/null
47+
exit 0
4748
}
4849
trap on_exit INT TERM
4950

cache/systemd/deis-cache.service

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,7 @@ EnvironmentFile=/etc/environment
66
TimeoutStartSec=20m
77
ExecStartPre=/bin/sh -c "docker history deis/cache >/dev/null || docker pull deis/cache:latest"
88
ExecStartPre=/bin/sh -c "docker inspect deis-cache >/dev/null && docker rm -f deis-cache || true"
9-
ExecStart=/usr/bin/docker run --name deis-cache -p 6379:6379 -e PUBLISH=6379 -e HOST=${COREOS_PRIVATE_IPV4} deis/cache
10-
ExecStop=/usr/bin/docker rm -f deis-cache
9+
ExecStart=/usr/bin/docker run --name deis-cache --rm -p 6379:6379 -e PUBLISH=6379 -e HOST=${COREOS_PRIVATE_IPV4} deis/cache
1110

1211
[Install]
1312
WantedBy=multi-user.target

controller/bin/boot

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ GUNICORN_PID=$!
6666
function on_exit() {
6767
kill -TERM $CELERY_PID $GUNICORN_PID
6868
wait $CELERY_PID $GUNICORN_PID 2>/dev/null
69+
exit 0
6970
}
7071
trap on_exit INT TERM
7172

controller/systemd/deis-controller.service

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,7 @@ EnvironmentFile=/etc/environment
88
TimeoutStartSec=20m
99
ExecStartPre=/bin/sh -c "docker history deis/controller >/dev/null || docker pull deis/controller:latest"
1010
ExecStartPre=/bin/sh -c "docker inspect deis-controller >/dev/null && docker rm -f deis-controller || true"
11-
ExecStart=/usr/bin/docker run --name deis-controller -p 8000:8000 -e PUBLISH=8000 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from=deis-logger deis/controller
12-
ExecStop=/usr/bin/docker rm -f deis-controller
11+
ExecStart=/usr/bin/docker run --name deis-controller --rm -p 8000:8000 -e PUBLISH=8000 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from=deis-logger deis/controller
1312

1413
[Install]
1514
WantedBy=multi-user.target

database/bin/boot

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,9 @@ SERVICE_PID=$!
5959

6060
# smart shutdown on SIGINT and SIGTERM
6161
function on_exit() {
62-
kill -TERM $SERVICE_PID
63-
wait $SERVICE_PID 2>/dev/null
62+
kill -TERM $SERVICE_PID
63+
wait $SERVICE_PID 2>/dev/null
64+
exit 0
6465
}
6566
trap on_exit INT TERM
6667

database/systemd/deis-database.service

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ TimeoutStartSec=20m
77
ExecStartPre=/bin/sh -c "docker history deis/database >/dev/null || docker pull deis/database:latest"
88
ExecStartPre=/bin/sh -c "docker inspect deis-database >/dev/null && docker rm -f deis-database || true"
99
ExecStartPre=/bin/sh -c "docker inspect deis-database-data >/dev/null 2>&1 || docker run --name deis-database-data -v /var/lib/postgresql deis/base true"
10-
ExecStart=/usr/bin/docker run --name deis-database -p 5432:5432 -e PUBLISH=5432 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-database-data deis/database
11-
ExecStop=/usr/bin/docker rm -f deis-database
10+
ExecStart=/usr/bin/docker run --name deis-database --rm -p 5432:5432 -e PUBLISH=5432 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-database-data deis/database
1211

1312
[Install]
1413
WantedBy=multi-user.target

logger/bin/boot

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ SERVICE_PID=$!
2626
function on_exit() {
2727
kill -TERM $SERVICE_PID
2828
wait $SERVICE_PID 2>/dev/null
29+
exit 0
2930
}
3031
trap on_exit INT TERM
3132

logger/systemd/deis-logger.service

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ TimeoutStartSec=20m
77
ExecStartPre=/bin/sh -c "docker history deis/logger >/dev/null || docker pull deis/logger:latest"
88
ExecStartPre=/bin/sh -c "docker inspect deis-logger >/dev/null && docker rm -f deis-logger || true"
99
ExecStartPre=/bin/sh -c "docker inspect deis-logger-data >/dev/null 2>&1 || docker run --name deis-logger-data -v /var/log/deis deis/base true"
10-
ExecStart=/usr/bin/docker run --name deis-logger -p 514:514/udp -e PUBLISH=514 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-logger-data deis/logger
11-
ExecStop=/usr/bin/docker rm -f deis-logger
10+
ExecStart=/usr/bin/docker run --name deis-logger --rm -p 514:514/udp -e PUBLISH=514 -e HOST=${COREOS_PRIVATE_IPV4} --volumes-from deis-logger-data deis/logger
1211

1312
[Install]
1413
WantedBy=multi-user.target

0 commit comments

Comments
 (0)