feat(scheduler): clean up stray pods when a release is deleted

Scaling an RC to 0 and deleting it does not always clear up all the active pods. This can happen if there is an image pull problem

Author: helgi

rootfs/api/models/release.py

@@ -183,8 +183,11 @@ def cleanup_old(self):
         )
 
         # Cleanup controllers
+        labels = {
+            'heritage': 'deis'
+        }
         controller_removal = []
-        controllers = self._scheduler._get_rcs(self.app.id).json()
+        controllers = self._scheduler._get_rcs(self.app.id, labels=labels).json()
         for controller in controllers['items']:
             current_version = controller['metadata']['labels']['version']
             # skip the latest release
@@ -208,6 +211,7 @@ def cleanup_old(self):
         # find stray env secrets to remove that may have been missed
         log_event(self.app, 'Cleaning up orphaned environment var secrets', level=logging.DEBUG)
         labels = {
+            'heritage': 'deis',
             'app': self.app.id,
             'type': 'env'
         }
@@ -220,6 +224,22 @@ def cleanup_old(self):
 
             self._scheduler._delete_secret(self.app.id, secret['metadata']['name'])
 
+        # Remove stray pods
+        labels = {
+            'heritage': 'deis'
+        }
+        pods = self._scheduler._get_pods(self.app.id, labels=labels).json()
+        for pod in pods['items']:
+            if self._scheduler._pod_deleted(pod):
+                continue
+
+            current_version = pod['metadata']['labels']['version']
+            # skip the latest release
+            if current_version == latest_version:
+                continue
+
+            self._scheduler._delete_pod(self.app.id, pod['metadata']['name'])
+
     def _delete_release_in_scheduler(self, namespace, version):
         """
         Deletes a specific release in k8s
@@ -228,13 +248,22 @@ def _delete_release_in_scheduler(self, namespace, version):
         secret that container the env var
         """
         labels = {
+            'heritage': 'deis',
             'app': namespace,
             'version': 'v{}'.format(version)
         }
-        controllers = self._scheduler._get_rcs(namespace, labels=labels)
-        for controller in controllers.json()['items']:
+        controllers = self._scheduler._get_rcs(namespace, labels=labels).json()
+        for controller in controllers['items']:
             self._scheduler._scale_rc(namespace, controller['metadata']['name'], 0)
             self._scheduler._delete_rc(namespace, controller['metadata']['name'])
+            # Remove stray pods
+            labels = controller['metadata']['labels']
+            pods = self._scheduler._get_pods(namespace, labels=labels).json()
+            for pod in pods['items']:
+                if self._scheduler._pod_deleted(pod):
+                    continue
+
+                self._scheduler._delete_pod(namespace, pod['metadata']['name'])
 
         # remove secret that contains env vars for the release
         try:

rootfs/scheduler/__init__.py

@@ -419,25 +419,39 @@ def deploy(self, namespace, name, image, command, **kwargs):  # noqa
             )
 
             # Remove new release of the RC
-            self._scale_rc(namespace, new_rc["metadata"]["name"], 0)
-            self._delete_rc(namespace, new_rc["metadata"]["name"])
+            self._cleanup_release(namespace, new_rc)
 
-            # Bring back old release if available of the RC
+            # If there was a previous release then bring that back
             if old_rc:
                 self._scale_rc(namespace, old_rc["metadata"]["name"], desired)
 
-            raise KubeException('{} (scheduler::deploy): {}'.format(name, e))
+            raise KubeException(str(e))
 
         # New release is live and kicking. Clean up old release
         if old_rc:
-            self._scale_rc(namespace, old_rc["metadata"]["name"], 0)
-            self._delete_rc(namespace, old_rc["metadata"]["name"])
+            self._cleanup_release(namespace, old_rc)
 
         # Make sure the application is routable and uses the correct port
         # Done after the fact to let initial deploy settle before routing
         # traffic to the application
         self._update_application_service(namespace, name, app_type, port, routable)
 
+    def _cleanup_release(self, namespace, controller):
+        """
+        Cleans up resources related to an application deployment
+        """
+        # Have the RC scale down pods and delete itself
+        self._scale_rc(namespace, controller['metadata']['name'], 0)
+        self._delete_rc(namespace, controller['metadata']['name'])
+
+        # Remove stray pods that the scale down will have missed (this can occassionally happen)
+        pods = self._get_pods(namespace, labels=controller['metadata']['labels']).json()
+        for pod in pods['items']:
+            if self._pod_deleted(pod):
+                continue
+
+            self._delete_pod(namespace, pod['metadata']['name'])
+
     def _update_application_service(self, namespace, name, app_type, port, routable=False):
         """Update application service with all the various required information"""
         try: