feat(deploy): support doing a larger number than 1 in, 1 out deployment

By default the deploy system will take the node count the application is deployed on (using tags), but can also look at DEPLOY_BRANCHES per application and a global Django django setting with the same name.

Example of the default flow is as follows: As a user I have 5 Nodes my application deploys to, the application has a scale of 17, this will lead to the following deploy cadence:
5, 5, 5, 2 (each item represents a in / out), so 5 pods being brought up 3x and then at the end 2 pods

Author: helgi

rootfs/api/models/app.py

@@ -341,6 +341,9 @@ def deploy(self, release):
             self.structure = self._default_structure(release)
             self.save()
 
+        # see if the app config has deploy batch preference, otherwise use global
+        batches = release.config.values.get('DEPLOY_BATCHES', settings.DEPLOY_BATCHES)
+
         # deploy application to k8s. Also handles initial scaling
         deploys = {}
         build_type = app_build_type(release)
@@ -357,7 +360,8 @@ def deploy(self, release):
                 'build_type': build_type,
                 'healthcheck': release.config.healthcheck(),
                 # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types
-                'routable': True if scale_type in ['web', 'cmd'] else False
+                'routable': True if scale_type in ['web', 'cmd'] else False,
+                'batches': batches
             }
 
         # Sort deploys so routable comes first

rootfs/deis/settings.py

@@ -268,6 +268,13 @@
 SLUG_BUILDER_IMAGE_PULL_POLICY = os.environ.get('SLUG_BUILDER_IMAGE_PULL_POLICY', "Always")  # noqa
 DOCKER_BUILDER_IMAGE_PULL_POLICY = os.environ.get('DOCKER_BUILDER_IMAGE_PULL_POLICY', "Always")  # noqa
 
+# Define a global default on how many pods to bring up and then
+# take down sequentially during a deploy
+# Defaults to None, the default is to deploy to as many nodes as
+# the application has been instructed to run on
+# Can also be overwritten on per app basis if desired
+DEPLOY_BATCHES = os.environ.get('DEPLOY_BATCHES', None)
+
 # How long k8s waits for a pod to finish work after a SIGTERM before sending SIGKILL
 KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(os.environ.get('KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 30))  # noqa
 

rootfs/scheduler/__init__.py

@@ -367,27 +367,40 @@ def deploy(self, namespace, name, image, command, **kwargs):  # noqa
             desired = kwargs['replicas']
             logger.debug('No prior RC could be found for {}-{}'.format(namespace, app_type))
 
+        # see if application or global deploy batches are defined
+        if not kwargs.get('batches', None):
+            # figure out how many nodes the application can go on
+            tags = kwargs.get('tags', {})
+            steps = len(self._get_nodes(labels=tags).json()['items'])
+        else:
+            steps = int(kwargs.get('batches'))
+
+        # figure out what kind of batches the deploy is done in - 1 in, 1 out or higher
+        if desired < steps:
+            # do it all in one go
+            batches = [desired]
+        else:
+            # figure out the stepped deploy count and then see if there is a leftover
+            batches = [steps for n in set(range(1, (desired + 1))) if n % steps == 0]
+            if desired - sum(batches) > 0:
+                batches.append(desired - sum(batches))
+
         try:
-            count = 1
-            while desired >= count:
+            count = 0
+            new_name = new_rc["metadata"]["name"]
+            for batch in batches:
+                count += batch
                 logger.debug('scaling release {} to {} out of final {}'.format(
-                    new_rc["metadata"]["name"], count, desired
-                ))
-                self._scale_rc(namespace, new_rc["metadata"]["name"], count)
-                logger.debug('scaled up pod number {} for {}'.format(
-                    count, new_rc["metadata"]["name"]
+                    new_name, count, desired
                 ))
+                self._scale_rc(namespace, new_name, count)
 
                 if old_rc:
+                    old_name = old_rc["metadata"]["name"]
                     logger.debug('scaling old release {} from original {} to {}'.format(
-                        old_rc["metadata"]["name"], desired, (desired-count))
+                        old_name, desired, (desired-count))
                     )
-                    self._scale_rc(namespace, old_rc["metadata"]["name"], (desired-count))
-                    logger.debug('scaled down pod number {} for {}'.format(
-                        count, old_rc["metadata"]["name"]
-                    ))
-
-                count += 1
+                    self._scale_rc(namespace, old_name, (desired-count))
         except Exception as e:
             # New release is broken. Clean up
             logger.error('Could not scale {} to {}. Deleting and going back to old release'.format(