Merge pull request #846 from helgi/scale_failure

fix(scheduler): if one RC fails to scale then ensure all other RCs are at the right level

Author: helgi

rootfs/api/models/app.py

@@ -331,15 +331,21 @@ def scale(self, user, structure):  # noqa
                     'Container type {} does not exist in application'.format(container_type))
 
         # merge current structure and the new items together
-        new_structure = self.structure.copy()
+        old_structure = self.structure
+        new_structure = old_structure.copy()
         new_structure.update(structure)
 
         if new_structure != self.structure:
             # save new structure to the database
             self.structure = new_structure
             self.save()
 
-            self._scale_pods(structure)
+            try:
+                self._scale_pods(structure)
+            except ServiceUnavailable:
+                # scaling failed, go back to old scaling numbers
+                self._scale_pods(old_structure)
+                raise
 
             msg = '{} scaled pods '.format(user.username) + ' '.join(
                 "{}={}".format(k, v) for k, v in list(structure.items()))

rootfs/scheduler/__init__.py

@@ -243,13 +243,8 @@ def scale(self, namespace, name, image, command, **kwargs):
                 logger.exception("Creating RC {} failed}".format(name))
                 raise
 
-        try:
-            self._scale_rc(namespace, name, replicas)
-        except KubeException:
-            logger.exception("Scaling failed for {}".format(name))
-            old = self.get_rc(namespace, name).json()
-            self._scale_rc(namespace, name, old['spec']['replicas'])
-            raise
+        # let the scale failure bubble up
+        self._scale_rc(namespace, name, replicas)
 
     def _build_pod_manifest(self, namespace, name, image, **kwargs):
         app_type = kwargs.get('app_type')