Merge pull request #3561 from iancoffey/gracefuller_shutdown

feat(contrib): optional graceful shutdown

Author: Matthew Fisher

contrib/coreos/user-data.example

@@ -43,6 +43,22 @@ coreos:
       Type=oneshot
       ExecStart=/usr/bin/systemctl stop update-engine.service
       ExecStartPost=/usr/bin/systemctl mask update-engine.service
+  - name: graceful-deis-shutdown.service
+    content: |
+      [Unit]
+      Description=Clean up
+      DefaultDependencies=no
+      After=fleet.service etcd.service docker.service docker.socket deis-store-admin.service deis-store-daemon.service deis-store-volume.service deis-store-monitor.service
+      Requires=fleet.service etcd.service deis-store-admin.service deis-store-daemon.service deis-store-volume.service docker.service docker.socket deis-store-monitor.service
+
+      [Install]
+      WantedBy=shutdown.target halt.target reboot.target
+
+      [Service]
+      ExecStop=/opt/bin/graceful-shutdown.sh --really
+      Type=oneshot
+      TimeoutSec=1200
+      RemainAfterExit=yes
   - name: install-deisctl.service
     command: start
     content: |
@@ -176,3 +192,61 @@ write_files:
     content: |
       [Coredump]
       Storage=none
+  - path: /opt/bin/graceful-shutdown.sh
+    permissions: '0755'
+    content: |
+      #!/usr/bin/bash
+      if [ "$1" != '--really' ]; then
+        echo "command must be run as: $0 --really"
+        exit 1
+      fi
+      # procedure requires the store-admin
+      ADMIN_RUNNING=$(docker inspect --format="{{ .State.Running }}" deis-store-admin)
+      if [ $? -eq 1 ] || [ "$ADMIN_RUNNING" == "false" ]; then
+        echo "deis-store-admin container is required for graceful shutdown"
+        exit 2
+      fi
+      set -e -x -o pipefail
+      # determine osd id
+      CURRENT_STATUS=$(/usr/bin/docker exec deis-store-admin ceph health | awk '{print $1}')
+      OSD_HOSTS=($(/usr/bin/etcdctl ls /deis/store/hosts/| awk -F'/' '{print $5}'))
+      for HOST in "${OSD_HOSTS[@]}"
+      do
+        PUBLIC_IP=$(fleetctl list-machines -fields="machine,ip" -full -no-legend| grep `cat /etc/machine-id` | awk '{print $2}')
+        if [ "$HOST" = "$PUBLIC_IP" ] ; then
+          OSD_ID=$(/usr/bin/etcdctl get /deis/store/osds/$PUBLIC_IP)
+          break
+        fi
+      done
+      # if we own an osd and its healthy, try to gracefully remove it
+      if [ ! -z "$OSD_ID" ] && [[ "$CURRENT_STATUS" == *"HEALTH_OK"* ]] && [ ${#OSD_HOSTS[@]} -gt "3" ]; then
+        /usr/bin/docker exec deis-store-admin ceph osd out $OSD_ID
+        sleep 30
+        TIMEWAITED=0
+        until [[ $(/usr/bin/docker exec deis-store-admin ceph health) == *"HEALTH_OK"* ]]
+        do
+          if [ $TIMEWAITED -gt "1200" ]
+          then
+            echo "ceph graceful removal timeout exceeded"
+            break
+          fi
+          echo "waiting" && sleep 5
+          TIMEWAITED=$((TIMEWAITED+5))
+        done
+        /usr/bin/docker stop deis-store-daemon
+        /usr/bin/docker exec deis-store-admin ceph osd crush remove osd.$OSD_ID
+        /usr/bin/docker exec deis-store-admin ceph auth del osd.$OSD_ID
+        /usr/bin/docker exec deis-store-admin ceph osd rm $OSD_ID
+        /usr/bin/etcdctl rm /deis/store/osds/$PUBLIC_IP
+        etcdctl rm /deis/store/hosts/$PUBLIC_IP && sleep 10
+        # remove ceph mon
+        /usr/bin/docker stop deis-store-monitor || true
+        /usr/bin/docker exec deis-store-admin ceph mon remove `hostname -f` # fixme
+        /usr/bin/docker stop deis-store-metadata || true
+      fi
+      NODE=$(curl -L http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`)
+      # remove from etcd cluster
+      if [ $NODE != 'null' ]; then
+        /usr/bin/curl -L -XDELETE http://127.0.0.1:7001/v2/admin/machines/`cat /etc/machine-id`
+      fi
+manage_etc_hosts: localhost

docs/managing_deis/add_remove_host.rst

@@ -284,3 +284,28 @@ This can be achieved by making a request to the etcd API. See `remove machines`_
 .. _`remove machines`: https://coreos.com/docs/distributed-configuration/etcd-api/#remove-machines
 .. _`removing monitors`: http://ceph.com/docs/hammer/rados/operations/add-or-rm-mons/#removing-monitors
 .. _`removing OSDs`: http://docs.ceph.com/docs/hammer/rados/operations/add-or-rm-osds/#removing-osds-manual
+
+Automatic Host Removal
+======================
+
+The ``contrib/user-data.example`` provides a unit which provides some experimental logic to clean-up a Deis
+node's Ceph and etcd membership before reboot, shutdown or halt events. Currently, the procedure will
+also be triggered if it is manually stopped via systemctl.
+
+The unit requires that the optional ``deis-store-admin`` component is installed.
+
+.. code-block:: console
+
+    root@deis-1:/# deisctl install store-admin
+    root@deis-1:/# deisctl start store-admin
+
+To enable the feature you must enable and start the unit:
+
+.. code-block:: console
+
+    root@deis-1:/# systemctl enable graceful-deis-shutdown
+    root@deis-1:/# systemctl start graceful-deis-shutdown
+
+The unit is now active and will now be stopped ahead of the store components, Docker and etcd.
+The script ``/opt/bin/graceful-shutdown.sh`` will determine if the Ceph cluster is healthy, and attempt
+to remove this node from the Deis store components if there are greater than 3 Ceph nodes in the cluster.