fix(scheduler): if one RC fails to scale then ensure all other RCs are at the right level

Previously only the failing RC would get fixed up but now the whole state will be reverted if one RC fails

Author: helgi

rootfs/api/models/app.py

@@ -331,15 +331,21 @@ def scale(self, user, structure):  # noqa
                     'Container type {} does not exist in application'.format(container_type))
 
         # merge current structure and the new items together
-        new_structure = self.structure.copy()
+        old_structure = self.structure
+        new_structure = old_structure.copy()
         new_structure.update(structure)
 
         if new_structure != self.structure:
             # save new structure to the database
             self.structure = new_structure
             self.save()
 
-            self._scale_pods(structure)
+            try:
+                self._scale_pods(structure)
+            except ServiceUnavailable:
+                # scaling failed, go back to old scaling numbers
+                self._scale_pods(old_structure)
+                raise
 
             msg = '{} scaled pods '.format(user.username) + ' '.join(
                 "{}={}".format(k, v) for k, v in list(structure.items()))

rootfs/scheduler/__init__.py

@@ -243,13 +243,8 @@ def scale(self, namespace, name, image, command, **kwargs):
                 logger.exception("Creating RC {} failed}".format(name))
                 raise
 
-        try:
-            self._scale_rc(namespace, name, replicas)
-        except KubeException:
-            logger.exception("Scaling failed for {}".format(name))
-            old = self.get_rc(namespace, name).json()
-            self._scale_rc(namespace, name, old['spec']['replicas'])
-            raise
+        # let the scale failure bubble up
+        self._scale_rc(namespace, name, replicas)
 
     def _build_pod_manifest(self, namespace, name, image, **kwargs):
         app_type = kwargs.get('app_type')