ref(release): cleanup old RCs and related kubernetes resources after deploy

Iterates through all available RCs beside the *latest*, scales them to 0 and deletes. Takes care of the release specific secret as well.
The secret has its own iteration as well in case anything happened on prior runs which made it into an orphan (no RC)

The biggest change here is that the code is not attempting to delete the secret during individual RC deploy but rather as part of the cleanup
after the deploy.

Fixes #540

Author: helgi

rootfs/api/models/app.py

@@ -384,6 +384,9 @@ def deploy(self, release):
                 log_event(self, err, logging.ERROR)
                 raise
 
+        # cleanup old releases from kubernetes
+        release.cleanup_old()
+
     def _default_structure(self, release):
         """Scale to default structure based on release type"""
         # if there is no SHA, assume a docker image is being promoted

rootfs/api/models/release.py

@@ -151,13 +151,7 @@ def rollback(self, user, version):
     def delete(self, *args, **kwargs):
         """Delete release DB record and any RCs from the affect release"""
         try:
-            labels = {
-                'app': self.app.id,
-                'version': 'v{}'.format(self.version)
-            }
-            controllers = self._scheduler._get_rcs(self.app.id, labels=labels)
-            for controller in controllers.json()['items']:
-                self._scheduler._delete_rc(self.app.id, controller['metadata']['name'])
+            self._delete_release_in_scheduler(self.app.id, self.version)
         except KubeHTTPException as e:
             # 404 means they were already cleaned up
             if e.status_code is not 404:
@@ -168,6 +162,68 @@ def delete(self, *args, **kwargs):
         finally:
             super(Release, self).delete(*args, **kwargs)
 
+    def cleanup_old(self):
+        """Cleanup all but the latest release from Kubernetes"""
+        latest_version = 'v{}'.format(self.version)
+        log_event(self.app, 'Cleaning up RCS for releases older than {} (latest)'.format(latest_version))  # noqa
+
+        # Cleanup controllers
+        controller_removal = []
+        controllers = self._scheduler._get_rcs(self.app.id).json()
+        for controller in controllers['items']:
+            current_version = controller['metadata']['labels']['version']
+            # skip the latest release
+            if current_version == latest_version:
+                continue
+
+            # aggregate versions together to removal all at once
+            if current_version not in controller_removal:
+                controller_removal.append(current_version)
+
+        if controller_removal:
+            log_event(self.app, 'Found the following versions to cleanup: {}'.format(', '.join(controller_removal)))  # noqa
+
+        for version in controller_removal:
+            self._delete_release_in_scheduler(self.app.id, version)
+
+        # find stray env secrets to remove that may have been missed
+        log_event(self.app, 'Cleaning up orphaned environment var secrets')
+        labels = {
+            'app': self.app.id,
+            'type': 'env'
+        }
+        secrets = self._scheduler._get_secrets(self.app.id, labels=labels).json()
+        for secret in secrets['items']:
+            current_version = secret['metadata']['labels']['version']
+            # skip the latest release
+            if current_version == latest_version:
+                continue
+
+            self._scheduler._delete_secret(self.app.id, secret['metadata']['name'])
+
+    def _delete_release_in_scheduler(self, namespace, version):
+        """
+        Deletes a specific release in k8s
+
+        Scale RCs to 0 then delete RCs and the version specific
+        secret that container the env var
+        """
+        labels = {
+            'app': namespace,
+            'version': 'v{}'.format(version)
+        }
+        controllers = self._scheduler._get_rcs(namespace, labels=labels)
+        for controller in controllers.json()['items']:
+            self._scheduler._scale_rc(namespace, controller['metadata']['name'], 0)
+            self._scheduler._delete_rc(namespace, controller['metadata']['name'])
+
+        # remove secret that contains env vars for the release
+        try:
+            secret_name = "{}-{}-env".format(namespace, version)
+            self._scheduler._delete_secret(namespace, secret_name)
+        except KubeHTTPException:
+            pass
+
     def save(self, *args, **kwargs):  # noqa
         if not self.summary:
             self.summary = ''

rootfs/scheduler/__init__.py

@@ -275,7 +275,10 @@
   "apiVersion": "$version",
   "metadata": {
     "name": "$name",
-    "namespace": "$id"
+    "namespace": "$id",
+    "labels": {
+      "app": "$id"
+    }
   },
   "type": "Opaque",
   "data": {}
@@ -386,31 +389,20 @@ def deploy(self, namespace, name, image, command, **kwargs):  # noqa
                 new_rc["metadata"]["name"], desired)
             )
 
-            # Remove new release
+            # Remove new release of the RC
             self._scale_rc(namespace, new_rc["metadata"]["name"], 0)
             self._delete_rc(namespace, new_rc["metadata"]["name"])
-            # remove secret that contains env vars for the release
-            try:
-                secret_name = "{}-{}-env".format(namespace, kwargs.get('version'))
-                self._delete_secret(namespace, secret_name)
-            except KubeHTTPException:
-                pass
 
-            # Bring back old release if available
+            # Bring back old release if available of the RC
             if old_rc:
                 self._scale_rc(namespace, old_rc["metadata"]["name"], desired)
 
             raise KubeException('{} (scheduler::deploy): {}'.format(name, e))
 
         # New release is live and kicking. Clean up old release
         if old_rc:
+            self._scale_rc(namespace, old_rc["metadata"]["name"], 0)
             self._delete_rc(namespace, old_rc["metadata"]["name"])
-            # remove secret that contains env vars for the release
-            secret_name = "{}-{}-env".format(namespace, old_rc['metadata']['labels']['version'])
-            try:
-                self._delete_secret(namespace, secret_name)
-            except KubeHTTPException:
-                pass
 
         # Make sure the application is routable and uses the correct port
         # Done after the fact to let initial deploy settle before routing
@@ -601,7 +593,11 @@ def _set_environment(self, data, namespace, **kwargs):
                 secret_name = "{}-{}-env".format(namespace, kwargs.get('version'))
                 self._get_secret(namespace, secret_name)
             except KubeHTTPException:
-                self._create_secret(namespace, secret_name, secrets_env)
+                labels = {
+                    'version': kwargs.get('version'),
+                    'type': 'env'
+                }
+                self._create_secret(namespace, secret_name, secrets_env, labels)
             else:
                 self._update_secret(namespace, secret_name, secrets_env)
 
@@ -1075,13 +1071,16 @@ def _get_secrets(self, namespace, **kwargs):
 
         return response
 
-    def _create_secret(self, namespace, name, data):
+    def _create_secret(self, namespace, name, data, labels={}):
         template = json.loads(string.Template(SECRET_TEMPLATE).substitute({
             "version": self.apiversion,
             "id": namespace,
             "name": name
         }))
 
+        # add in any additional label info
+        template['metadata']['labels'].update(labels)
+
         for key, value in data.items():
             value = value if isinstance(value, bytes) else bytes(value, 'UTF-8')
             item = base64.b64encode(value).decode(encoding='UTF-8')