feat(*): add Deployments support behind a feature flag

This adds Deployment object support (replaces ReplicationController) when a feature flag is enabled. This can be done by either settings a global ENV Var on the Controller or setting a config per app. The variable is DEIS_KUBERNETES_DEPLOYMENT=1

A few new features (Deployments only) have been introduced or specifically expose Deployment functionalities:
* KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT (global and per app setting) specifies how many revisions (ReplicaSets) to keep
* Stop concurrent deployments (triggered by push, pull, config:set and so on)

If Deployments is enabled on an existing app then a whole new copy will be spun up (behind the scenes) as Deployment and then the old ReplicationController is spun down. This is a one way conversion.

Note that pod naming has changed due to how Deployments adds a hash to the name, among other things.

Author: helgi

rootfs/api/exceptions.py

@@ -39,7 +39,7 @@ def custom_exception_handler(exc, context):
         set_rollback()
         return Response({'detail': 'Server Error'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
 
-    # log a few different types of exception instead of use APIException
+    # log a few different types of exception instead of using APIException
     if isinstance(exc, (DeisException, ServiceUnavailable, HealthcheckException)):
         logging.exception(exc.__cause__, exc_info=exc)
 

rootfs/api/models/app.py

@@ -121,9 +121,16 @@ def __str__(self):
     def _get_job_id(self, container_type):
         app = self.id
         release = self.release_set.latest()
+
+        # see if there is a global or app specific setting to specify Deployments usage
+        deployments = bool(release.config.values.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
+
+        # deployments does not need version in the job
+        if deployments:
+            return "{app}-{container_type}".format(**locals())
+
         version = "v{}".format(release.version)
-        job_id = "{app}-{version}-{container_type}".format(**locals())
-        return job_id
+        return "{app}-{version}-{container_type}".format(**locals())
 
     def _get_command(self, container_type):
         try:
@@ -237,35 +244,48 @@ def delete(self, *args, **kwargs):
 
     def restart(self, **kwargs):  # noqa
         """
-        Restart found pods by deleting them (RC will recreate).
-        Wait until they are all drained away and RC has gotten to a good state
+        Restart found pods by deleting them (RC / Deployment will recreate).
+        Wait until they are all drained away and RC / Deployment has gotten to a good state
         """
         try:
-            # Resolve single pod name if short form (worker-asdfg) is passed
-            if 'name' in kwargs and kwargs['name'].count('-') == 1:
-                if 'release' not in kwargs or kwargs['release'] is None:
-                    release = self.release_set.latest()
-                else:
-                    release = self.release_set.get(version=kwargs['release'])
+            if kwargs.get('release', None) is None:
+                release = self.release_set.latest()
+            else:
+                release = self.release_set.get(version=kwargs['release'])
 
-                version = "v{}".format(release.version)
-                kwargs['name'] = '{}-{}-{}'.format(kwargs['id'], version, kwargs['name'])
+            # see if there is a global or app specific setting to specify Deployments usage
+            deployments = bool(release.config.values.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
 
-            # Iterate over RCs to get total desired count if not a single item
+            if deployments:
+                # Resolve single pod name if short form (cmd-1269180282-1nyfz) is passed
+                if 'name' in kwargs and kwargs['name'].count('-') == 2:
+                    kwargs['name'] = '{}-{}'.format(kwargs['id'], kwargs['name'])
+            else:
+                # Resolve single pod name if short form (worker-asdfg) is passed
+                if 'name' in kwargs and kwargs['name'].count('-') == 1:
+                    version = "v{}".format(release.version)
+                    kwargs['name'] = '{}-{}-{}'.format(kwargs['id'], version, kwargs['name'])
+
+            # Iterate over RCs / RSs to get total desired count if not a single item
             desired = 1
             if 'name' not in kwargs:
                 desired = 0
                 labels = self._scheduler_filter(**kwargs)
-                controllers = self._scheduler.get_rcs(kwargs['id'], labels=labels).json()['items']
-                for controller in controllers:
+                # fetch RS (which represent Deployments) / RCs
+                if deployments:
+                    controllers = self._scheduler.get_replicasets(kwargs['id'], labels=labels)
+                else:
+                    controllers = self._scheduler.get_rcs(kwargs['id'], labels=labels)
+
+                for controller in controllers.json()['items']:
                     desired += controller['spec']['replicas']
         except KubeException:
             # Nothing was found
             return []
 
         try:
             for pod in self.list_pods(**kwargs):
-                # This function verifies the delete. Gives pod 30 seconds
+                # This function verifies the delete
                 self._scheduler.delete_pod(self.id, pod['name'])
         except Exception as e:
             err = "warning, some pods failed to stop:\n{}".format(str(e))
@@ -380,6 +400,13 @@ def scale(self, user, structure):  # noqa
     def _scale_pods(self, scale_types):
         release = self.release_set.latest()
         envs = release.config.values
+
+        # see if the app config has deploy batch preference, otherwise use global
+        batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES)
+
+        # see if there is a global or app specific setting to specify Deployments usage
+        deployments = bool(envs.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
+
         for scale_type, replicas in scale_types.items():
             # only web / cmd are routable
             # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types
@@ -400,7 +427,10 @@ def _scale_pods(self, scale_types):
                 'app_type': scale_type,
                 'build_type': release.build.type,
                 'healthcheck': release.config.healthcheck,
-                'routable': routable
+                'routable': routable,
+                'deployments': deployments,
+                'deploy_batches': batches,
+                'deploy_timeout': 120,  # 2 minutes
             }
 
             command = self._get_command(scale_type)
@@ -417,8 +447,12 @@ def _scale_pods(self, scale_types):
                 self.log(err, logging.ERROR)
                 raise ServiceUnavailable(err) from e
 
-    def deploy(self, release):
-        """Deploy a new release to this application"""
+    def deploy(self, release, force_deploy=False):
+        """
+        Deploy a new release to this application
+
+        force_deploy can be used when a deployment is broken, such as for Rollback
+        """
         if release.build is None:
             raise DeisException('No build associated with this release')
 
@@ -432,6 +466,11 @@ def deploy(self, release):
         # see if the app config has deploy batch preference, otherwise use global
         batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES)
 
+        # see if there is a global or app specific setting to specify Deployments usage
+        deployments = bool(release.config.values.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
+
+        deployment_history = release.config.values.get('KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT)  # noqa
+
         # deploy application to k8s. Also handles initial scaling
         deploys = {}
         envs = release.config.values
@@ -457,14 +496,23 @@ def deploy(self, release):
                 'build_type': release.build.type,
                 'healthcheck': release.config.healthcheck,
                 'routable': routable,
-                'deploy_batches': batches
+                'deploy_batches': batches,
+                'deploy_timeout': 120,  # 2 minutes
+                'deployment_history_limit': deployment_history,
+                'deployments': deployments,
+                'release_summary': release.summary
             }
 
         # Sort deploys so routable comes first
         deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
 
         for scale_type, kwargs in deploys.items():
             try:
+                # Is there an existing deployment in progress?
+                name = self._get_job_id(scale_type)
+                if not force_deploy and release.deployment_in_progress(self.id, name):
+                    raise AlreadyExists('Deployment for {} is already in progress'.format(name))
+
                 self._scheduler.deploy(
                     namespace=self.id,
                     name=self._get_job_id(scale_type),
@@ -484,8 +532,8 @@ def deploy(self, release):
                 self.log(err, logging.ERROR)
                 raise ServiceUnavailable(err) from e
 
-        # cleanup old releases from kubernetes
-        release.cleanup_old()
+        # cleanup old release objects from kubernetes
+        release.cleanup_old(deployments)
 
     def _default_structure(self, release):
         """Scale to default structure based on release type"""
@@ -668,8 +716,9 @@ def list_pods(self, *args, **kwargs):
 
             data = []
             for p in pods:
+                labels = p['metadata']['labels']
                 # specifically ignore run pods
-                if p['metadata']['labels']['type'] == 'run':
+                if labels['type'] == 'run':
                     continue
 
                 state = str(self._scheduler.pod_state(p))
@@ -685,8 +734,8 @@ def list_pods(self, *args, **kwargs):
                 item = Pod()
                 item['name'] = p['metadata']['name']
                 item['state'] = state
-                item['release'] = p['metadata']['labels']['version']
-                item['type'] = p['metadata']['labels']['type']
+                item['release'] = labels['version']
+                item['type'] = labels['type']
                 if 'startTime' in p['status']:
                     started = p['status']['startTime']
                 else:
@@ -697,7 +746,6 @@ def list_pods(self, *args, **kwargs):
 
             # sorting so latest start date is first
             data.sort(key=lambda x: x['started'], reverse=True)
-
             return data
         except KubeHTTPException as e:
             pass
@@ -707,7 +755,7 @@ def list_pods(self, *args, **kwargs):
             raise ServiceUnavailable(err) from e
 
     def _scheduler_filter(self, **kwargs):
-        labels = {'app': self.id}
+        labels = {'app': self.id, 'heritage': 'deis'}
 
         # always supply a version, either latest or a specific one
         if 'release' not in kwargs or kwargs['release'] is None:

rootfs/api/models/release.py

@@ -225,7 +225,7 @@ def rollback(self, user, version=None):
             )
 
             if self.build is not None:
-                self.app.deploy(new_release)
+                self.app.deploy(new_release, force_deploy=True)
             return new_release
         except Exception as e:
             if 'new_release' in locals():
@@ -246,11 +246,11 @@ def delete(self, *args, **kwargs):
         finally:
             super(Release, self).delete(*args, **kwargs)
 
-    def cleanup_old(self):
+    def cleanup_old(self, deployment=False):
         """Cleanup all but the latest release from Kubernetes"""
         latest_version = 'v{}'.format(self.version)
         self.app.log(
-            'Cleaning up RCS for releases older than {} (latest)'.format(latest_version),
+            'Cleaning up RCs for releases older than {} (latest)'.format(latest_version),
             level=logging.DEBUG
         )
 
@@ -282,7 +282,7 @@ def cleanup_old(self):
         labels = {
             'heritage': 'deis',
             'app': self.app.id,
-            'type': 'env'
+            'type': 'env',
         }
         secrets = self._scheduler.get_secrets(self.app.id, labels=labels).json()
         for secret in secrets['items']:
@@ -307,18 +307,58 @@ def cleanup_old(self):
 
             self._scheduler.delete_pod(self.app.id, pod['metadata']['name'])
 
+        if deployment:
+            self._cleanup_deployment_secrets_and_configs(self.app.id)
+
+    def _cleanup_deployment_secrets_and_configs(self, namespace):
+        """
+        Clean up any environment secrets (and in the future ConfigMaps) that
+        are tied to a release Deployments no longer track
+
+        This is done by checking the available ReplicaSets and only removing
+        objects not attached to anything. This will allow releases done outside
+        of Deis Controller
+        """
+
+        # Find all ReplicaSets
+        versions = []
+        labels = {'heritage': 'deis', 'app': namespace}
+        replicasets = self._scheduler.get_replicasets(namespace, labels=labels).json()
+        for replicaset in replicasets['items']:
+            if (
+                'version' not in replicaset['metadata']['labels'] or
+                replicaset['metadata']['labels']['version'] in versions
+            ):
+                continue
+
+            versions.append(replicaset['metadata']['labels']['version'])
+
+        # find all env secrets not owned by any replicaset
+        labels = {
+            'heritage': 'deis',
+            'app': namespace,
+            'type': 'env',
+            # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
+            'version__notin': versions
+        }
+        self.app.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
+        secrets = self._scheduler.get_secrets(namespace, labels=labels).json()
+        for secret in secrets['items']:
+            self._scheduler.delete_secret(namespace, secret['metadata']['name'])
+
     def _delete_release_in_scheduler(self, namespace, version):
         """
-        Deletes a specific release in k8s
+        Deletes a specific release in k8s based on ReplicationController
 
-        Scale RCs to 0 then delete RCs and the version specific
+        Scale RCs to 0 then delete RCs / Deployments and the version specific
         secret that container the env var
         """
         labels = {
             'heritage': 'deis',
             'app': namespace,
-            'version': 'v{}'.format(version)
+            'version': version
         }
+
         controllers = self._scheduler.get_rcs(namespace, labels=labels).json()
         for controller in controllers['items']:
             self._scheduler.cleanup_release(namespace, controller)
@@ -330,6 +370,20 @@ def _delete_release_in_scheduler(self, namespace, version):
         except KubeHTTPException:
             pass
 
+    def deployment_in_progress(self, namespace, name):
+        # see if there is a global or app specific setting to specify Deployments usage
+        deployments = bool(self.config.values.get('DEIS_KUBERNETES_DEPLOYMENTS', settings.DEIS_KUBERNETES_DEPLOYMENTS))  # noqa
+        if not deployments:
+            return False
+
+        try:
+            ready, _ = self._scheduler.are_deployment_replicas_ready(namespace, name)
+            return not ready
+        except KubeHTTPException as e:
+            # Deployment doesn't exist
+            if e.response.status_code == 404:
+                return False
+
     def save(self, *args, **kwargs):  # noqa
         if not self.summary:
             self.summary = ''

rootfs/api/settings/production.py

@@ -265,6 +265,11 @@
 # Can also be overwritten on per app basis if desired
 DEIS_DEPLOY_BATCHES = os.environ.get('DEIS_DEPLOY_BATCHES', None)
 
+# If the k8s Deployments object should be used instead of ReplicationController
+DEIS_KUBERNETES_DEPLOYMENTS = bool(os.environ.get('DEIS_KUBERNETES_DEPLOYMENTS', False))
+
+KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT = os.environ.get('KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', None)  # noqa
+
 # How long k8s waits for a pod to finish work after a SIGTERM before sending SIGKILL
 KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(os.environ.get('KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 30))  # noqa
 

rootfs/api/tests/test_build.py

@@ -123,7 +123,10 @@ def test_build_default_containers(self, mock_requests):
         self.assertEqual(container['type'], 'cmd')
         self.assertEqual(container['release'], 'v2')
         # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-v2-cmd-[a-z0-9]{5}')
+        if settings.DEIS_KUBERNETES_DEPLOYMENTS:
+            self.assertRegex(container['name'], app_id + '-cmd-[0-9]{8,10}-[a-z0-9]{5}')
+        else:
+            self.assertRegex(container['name'], app_id + '-v2-cmd-[a-z0-9]{5}')
 
         # start with a new app
         url = '/v2/apps'
@@ -148,7 +151,10 @@ def test_build_default_containers(self, mock_requests):
         self.assertEqual(container['type'], 'cmd')
         self.assertEqual(container['release'], 'v2')
         # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-v2-cmd-[a-z0-9]{5}')
+        if settings.DEIS_KUBERNETES_DEPLOYMENTS:
+            self.assertRegex(container['name'], app_id + '-cmd-[0-9]{8,10}-[a-z0-9]{5}')
+        else:
+            self.assertRegex(container['name'], app_id + '-v2-cmd-[a-z0-9]{5}')
 
         # start with a new app
         url = '/v2/apps'
@@ -177,7 +183,10 @@ def test_build_default_containers(self, mock_requests):
         self.assertEqual(container['type'], 'cmd')
         self.assertEqual(container['release'], 'v2')
         # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-v2-cmd-[a-z0-9]{5}')
+        if settings.DEIS_KUBERNETES_DEPLOYMENTS:
+            self.assertRegex(container['name'], app_id + '-cmd-[0-9]{8,10}-[a-z0-9]{5}')
+        else:
+            self.assertRegex(container['name'], app_id + '-v2-cmd-[a-z0-9]{5}')
 
         # start with a new app
         url = '/v2/apps'
@@ -206,7 +215,10 @@ def test_build_default_containers(self, mock_requests):
         self.assertEqual(container['type'], 'web')
         self.assertEqual(container['release'], 'v2')
         # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-v2-web-[a-z0-9]{5}')
+        if settings.DEIS_KUBERNETES_DEPLOYMENTS:
+            self.assertRegex(container['name'], app_id + '-web-[0-9]{8,10}-[a-z0-9]{5}')
+        else:
+            self.assertRegex(container['name'], app_id + '-v2-web-[a-z0-9]{5}')
 
     def test_build_str(self, mock_requests):
         """Test the text representation of a build."""