ref(scheduler): use observedGeneration instead of resourceVersion for RC readiness

https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata explains the resourceVersion and generation pretty well

resourceVersion is the internal version representation where as generation deals with desired state

Some resources report the observedGeneration, which is the generation most recently observed by the component responsible for acting upon changes to the desired state of the resource. This can be used, for instance, to ensure that the reported status reflects the most recent desired status.

Author: helgi

rootfs/scheduler/__init__.py

@@ -930,20 +930,7 @@ def _scale_rc(self, namespace, name, desired):
             logger.info("scaling RC {} in Namespace {} from {} to {} replicas".format(name, namespace, current, desired))  # noqa
 
             self._update_rc(namespace, name, rc)
-
-            resource_ver = rc['metadata']['resourceVersion']
-            logger.info("waiting for RC {} to get a newer resource version than {} (30s timeout)".format(name, resource_ver))  # noqa
-            for waited in range(30):
-                js_template = self._get_rc(namespace, name).json()
-                if js_template["metadata"]["resourceVersion"] != resource_ver:
-                    break
-
-                if waited > 0 and (waited % 10) == 0:
-                    logger.info("waited {}s so far for a new resource version".format(waited))
-
-                time.sleep(1)
-
-            logger.info("RC {} has a new resource version {}".format(name, js_template["metadata"]["resourceVersion"]))  # noqa
+            self._wait_for_rc_ready(namespace, name)
 
         # Get application container
         container_name = '{}-{}'.format(
@@ -1032,16 +1019,21 @@ def _create_rc(self, namespace, name, image, command, **kwargs):  # noqa
 
     def _wait_for_rc_ready(self, namespace, name):
         """
-        Waits for status/observedGeneration and metadata/generation to match
-        Indicates RC is ready
+        Looks at status/observedGeneration and metadata/generation and
+        waits for observedGeneration >= generation to happen, indicates RC is ready
+
+        More information is also available at:
+        https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
+        logger.debug("waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name))  # noqa
         for _ in range(30):
             try:
                 rc = self._get_rc(namespace, name).json()
                 if (
                     "observedGeneration" in rc["status"] and
-                    rc["metadata"]["generation"] == rc["status"]["observedGeneration"]
+                    rc["status"]["observedGeneration"] >= rc["metadata"]["generation"]
                 ):
+                    logger.debug("ReplicationController {} got a newer generation (30s timeout)".format(name))  # noqa
                     break
 
                 time.sleep(1)