Merge pull request #633 from helgi/delete_rc_app

ref(release): cleanup old RCs and related kubernetes resources after deploy

Author: helgi

rootfs/api/models/app.py

@@ -384,6 +384,9 @@ def deploy(self, release):
                 log_event(self, err, logging.ERROR)
                 raise
 
+        # cleanup old releases from kubernetes
+        release.cleanup_old()
+
     def _default_structure(self, release):
         """Scale to default structure based on release type"""
         # if there is no SHA, assume a docker image is being promoted

rootfs/api/models/release.py

@@ -151,13 +151,7 @@ def rollback(self, user, version):
     def delete(self, *args, **kwargs):
         """Delete release DB record and any RCs from the affect release"""
         try:
-            labels = {
-                'app': self.app.id,
-                'version': 'v{}'.format(self.version)
-            }
-            controllers = self._scheduler._get_rcs(self.app.id, labels=labels)
-            for controller in controllers.json()['items']:
-                self._scheduler._delete_rc(self.app.id, controller['metadata']['name'])
+            self._delete_release_in_scheduler(self.app.id, self.version)
         except KubeHTTPException as e:
             # 404 means they were already cleaned up
             if e.status_code is not 404:
@@ -168,6 +162,68 @@ def delete(self, *args, **kwargs):
         finally:
             super(Release, self).delete(*args, **kwargs)
 
+    def cleanup_old(self):
+        """Cleanup all but the latest release from Kubernetes"""
+        latest_version = 'v{}'.format(self.version)
+        log_event(self.app, 'Cleaning up RCS for releases older than {} (latest)'.format(latest_version))  # noqa
+
+        # Cleanup controllers
+        controller_removal = []
+        controllers = self._scheduler._get_rcs(self.app.id).json()
+        for controller in controllers['items']:
+            current_version = controller['metadata']['labels']['version']
+            # skip the latest release
+            if current_version == latest_version:
+                continue
+
+            # aggregate versions together to removal all at once
+            if current_version not in controller_removal:
+                controller_removal.append(current_version)
+
+        if controller_removal:
+            log_event(self.app, 'Found the following versions to cleanup: {}'.format(', '.join(controller_removal)))  # noqa
+
+        for version in controller_removal:
+            self._delete_release_in_scheduler(self.app.id, version)
+
+        # find stray env secrets to remove that may have been missed
+        log_event(self.app, 'Cleaning up orphaned environment var secrets')
+        labels = {
+            'app': self.app.id,
+            'type': 'env'
+        }
+        secrets = self._scheduler._get_secrets(self.app.id, labels=labels).json()
+        for secret in secrets['items']:
+            current_version = secret['metadata']['labels']['version']
+            # skip the latest release
+            if current_version == latest_version:
+                continue
+
+            self._scheduler._delete_secret(self.app.id, secret['metadata']['name'])
+
+    def _delete_release_in_scheduler(self, namespace, version):
+        """
+        Deletes a specific release in k8s
+
+        Scale RCs to 0 then delete RCs and the version specific
+        secret that container the env var
+        """
+        labels = {
+            'app': namespace,
+            'version': 'v{}'.format(version)
+        }
+        controllers = self._scheduler._get_rcs(namespace, labels=labels)
+        for controller in controllers.json()['items']:
+            self._scheduler._scale_rc(namespace, controller['metadata']['name'], 0)
+            self._scheduler._delete_rc(namespace, controller['metadata']['name'])
+
+        # remove secret that contains env vars for the release
+        try:
+            secret_name = "{}-{}-env".format(namespace, version)
+            self._scheduler._delete_secret(namespace, secret_name)
+        except KubeHTTPException:
+            pass
+
     def save(self, *args, **kwargs):  # noqa
         if not self.summary:
             self.summary = ''

rootfs/scheduler/__init__.py

@@ -275,7 +275,10 @@
   "apiVersion": "$version",
   "metadata": {
     "name": "$name",
-    "namespace": "$id"
+    "namespace": "$id",
+    "labels": {
+      "app": "$id"
+    }
   },
   "type": "Opaque",
   "data": {}
@@ -386,31 +389,20 @@ def deploy(self, namespace, name, image, command, **kwargs):  # noqa
                 new_rc["metadata"]["name"], desired)
             )
 
-            # Remove new release
+            # Remove new release of the RC
             self._scale_rc(namespace, new_rc["metadata"]["name"], 0)
             self._delete_rc(namespace, new_rc["metadata"]["name"])
-            # remove secret that contains env vars for the release
-            try:
-                secret_name = "{}-{}-env".format(namespace, kwargs.get('version'))
-                self._delete_secret(namespace, secret_name)
-            except KubeHTTPException:
-                pass
 
-            # Bring back old release if available
+            # Bring back old release if available of the RC
             if old_rc:
                 self._scale_rc(namespace, old_rc["metadata"]["name"], desired)
 
             raise KubeException('{} (scheduler::deploy): {}'.format(name, e))
 
         # New release is live and kicking. Clean up old release
         if old_rc:
+            self._scale_rc(namespace, old_rc["metadata"]["name"], 0)
             self._delete_rc(namespace, old_rc["metadata"]["name"])
-            # remove secret that contains env vars for the release
-            secret_name = "{}-{}-env".format(namespace, old_rc['metadata']['labels']['version'])
-            try:
-                self._delete_secret(namespace, secret_name)
-            except KubeHTTPException:
-                pass
 
         # Make sure the application is routable and uses the correct port
         # Done after the fact to let initial deploy settle before routing
@@ -601,7 +593,11 @@ def _set_environment(self, data, namespace, **kwargs):
                 secret_name = "{}-{}-env".format(namespace, kwargs.get('version'))
                 self._get_secret(namespace, secret_name)
             except KubeHTTPException:
-                self._create_secret(namespace, secret_name, secrets_env)
+                labels = {
+                    'version': kwargs.get('version'),
+                    'type': 'env'
+                }
+                self._create_secret(namespace, secret_name, secrets_env, labels)
             else:
                 self._update_secret(namespace, secret_name, secrets_env)
 
@@ -1075,13 +1071,16 @@ def _get_secrets(self, namespace, **kwargs):
 
         return response
 
-    def _create_secret(self, namespace, name, data):
+    def _create_secret(self, namespace, name, data, labels={}):
         template = json.loads(string.Template(SECRET_TEMPLATE).substitute({
             "version": self.apiversion,
             "id": namespace,
             "name": name
         }))
 
+        # add in any additional label info
+        template['metadata']['labels'].update(labels)
+
         for key, value in data.items():
             value = value if isinstance(value, bytes) else bytes(value, 'UTF-8')
             item = base64.b64encode(value).decode(encoding='UTF-8')