feat(scheduler): allow setting gracefulTerminationPeriod and account for in terminations

Allows users to set how long k8s allows pods to stay alive after a SIGTERM before they send SIGKILL - Defaults to 30 (like k8s) but lets users set a lower value for faster deploys if the apps can handle it.

Reduces the pod termination wait to the new setting and uses the existance of "deletionTimestamp" as an indicator if a pod is Terminating, also checks if the pod is past the graceful period and hides it from the pod list via ps:list

Fixes #620
Fixes #621
Fixes #631

Author: helgi

rootfs/api/models/app.py

@@ -582,9 +582,28 @@ def list_pods(self, *args, **kwargs):
                 if p['metadata']['labels']['type'] == 'run':
                     continue
 
+                state = self._scheduler.resolve_state(p).name
+
+                # follows kubelete convention - these are hidden unless show-all is set
+                if state in ['down', 'crashed']:
+                    continue
+
+                # hide pods that are past their graceful termination
+                # https://github.com/kubernetes/kubernetes/blob/release-1.2/docs/devel/api-conventions.md#metadata
+                # http://kubernetes.io/docs/user-guide/pods/#termination-of-pods
+                if 'deletionTimestamp' in p['metadata']:
+                    deletion = datetime.strptime(
+                        p['metadata']['deletionTimestamp'],
+                        settings.DEIS_DATETIME_FORMAT
+                    )
+
+                    # past the graceful deletion period
+                    if deletion < datetime.utcnow():
+                        continue
+
                 item = Pod()
                 item['name'] = p['metadata']['name']
-                item['state'] = self._scheduler.resolve_state(p).name
+                item['state'] = state
                 item['release'] = p['metadata']['labels']['version']
                 item['type'] = p['metadata']['labels']['type']
                 if 'startTime' in p['status']:

rootfs/deis/settings.py

@@ -268,6 +268,9 @@
 SLUG_BUILDER_IMAGE_PULL_POLICY = os.environ.get('SLUG_BUILDER_IMAGE_PULL_POLICY', "Always")  # noqa
 DOCKER_BUILDER_IMAGE_PULL_POLICY = os.environ.get('DOCKER_BUILDER_IMAGE_PULL_POLICY', "Always")  # noqa
 
+# How long k8s waits for a pod to finish work after a SIGTERM before sending SIGKILL
+KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(os.environ.get('KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 30))  # noqa
+
 # registry settings
 REGISTRY_HOST = os.environ.get('DEIS_REGISTRY_SERVICE_HOST', '127.0.0.1')
 REGISTRY_PORT = os.environ.get('DEIS_REGISTRY_SERVICE_PORT', 5000)

rootfs/scheduler/__init__.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import json
 import logging
 import os
@@ -63,13 +64,14 @@
       }
     ],
     "volumes":[
-    {
+      {
         "name":"objectstorage-keyfile",
         "secret":{
         "secretName":"objectstorage-keyfile"
         }
-    }
+      }
     ],
+    "terminationGracePeriodSeconds": "$terminationGracePeriodSeconds",
     "restartPolicy": "Never"
   }
 }
@@ -90,6 +92,7 @@
         "env": []
       }
     ],
+    "terminationGracePeriodSeconds": "$terminationGracePeriodSeconds",
     "restartPolicy": "Never"
   }
 }
@@ -126,6 +129,7 @@
         }
       },
       "spec": {
+        "terminationGracePeriodSeconds": "$terminationGracePeriodSeconds",
         "containers": [
           {
             "name": "$containername",
@@ -181,6 +185,7 @@
         }
       },
       "spec": {
+        "terminationGracePeriodSeconds": "$terminationGracePeriodSeconds",
         "containers": [
           {
             "name": "$containername",
@@ -217,11 +222,11 @@
             }
             ],
             "volumeMounts":[
-            {
+              {
                 "name":"objectstorage-keyfile",
                 "mountPath":"/var/run/secrets/deis/objectstore/creds",
                 "readOnly":true
-            }
+              }
             ]
           }
         ],
@@ -508,7 +513,8 @@ def run(self, namespace, name, image, entrypoint, command, **kwargs):
             'version': self.apiversion,
             'image': imgurl,
             'image_pull_policy': settings.DOCKER_BUILDER_IMAGE_PULL_POLICY,
-            'storagetype': os.getenv("APP_STORAGE")
+            'storagetype': os.getenv("APP_STORAGE"),
+            'terminationGracePeriodSeconds': settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS  # noqa
         }
 
         if entrypoint == '/runner/init':
@@ -796,13 +802,33 @@ def _get_rcs(self, namespace, **kwargs):
         return response
 
     def _wait_until_pods_terminate(self, namespace, labels, current, desired):
-        delta = current - desired
+        """Wait until all the desired pods are terminated"""
+        # http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_podspec
+        # https://github.com/kubernetes/kubernetes/blob/release-1.2/docs/devel/api-conventions.md#metadata
+        # http://kubernetes.io/docs/user-guide/pods/#termination-of-pods
 
-        logger.debug("waiting for {} pods in {} namespace to be terminated (120s timeout)".format(delta, namespace))  # noqa
-        for waited in range(120):
+        timeout = settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS
+        delta = current - desired
+        logger.debug("waiting for {} pods in {} namespace to be terminated ({}s timeout)".format(delta, namespace, timeout))  # noqa
+        for waited in range(timeout):
             pods = self._get_pods(namespace, labels=labels).json()
             count = len(pods['items'])
 
+            # see if any pods are past their terminationGracePeriodsSeconds (as in stuck)
+            # seems to be a problem in k8s around that:
+            # https://github.com/kubernetes/kubernetes/search?q=terminating&type=Issues
+            # these will be eventually GC'ed by k8s, ignoring them for now
+            for pod in pods['items']:
+                if 'deletionTimestamp' in pod['metadata']:
+                    deletion = datetime.strptime(
+                        pod['metadata']['deletionTimestamp'],
+                        settings.DEIS_DATETIME_FORMAT
+                    )
+
+                    # past the graceful deletion period
+                    if deletion < datetime.utcnow():
+                        count -= 1
+
             # stop when all pods are terminated as expected
             if count == desired:
                 break
@@ -909,6 +935,7 @@ def _create_rc(self, namespace, name, image, command, **kwargs):  # noqa
             "storagetype": storageType,
             "mHost": os.getenv("DEIS_MINIO_SERVICE_HOST"),
             "mPort": os.getenv("DEIS_MINIO_SERVICE_PORT"),
+            "terminationGracePeriodSeconds": settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS  # noqa
         }
 
         # Check if it is a slug builder image.
@@ -1313,9 +1340,16 @@ def _pod_readiness_status(self, pod):
                 if not container['ready']:
                     if 'running' in container['state'].keys():
                         return 'Starting'
-                    elif 'terminated' in container['state'].keys():
+                    elif (
+                        'terminated' in container['state'].keys() or
+                        'deletionTimestamp' in pod['metadata']
+                    ):
                         return 'Terminating'
                 else:
+                    # See if k8s is in Terminating state
+                    if 'deletionTimestamp' in pod['metadata']:
+                        return 'Terminating'
+
                     return 'Running'
 
         # Seems like the most sensible default