Merge pull request #2072 from gabrtv/retry

Retry Fleet container scheduling operations

Author: Gabriel Monroy

controller/scheduler/coreos.py

@@ -11,6 +11,7 @@
 
 MATCH = re.compile(
     '(?P<app>[a-z0-9-]+)_?(?P<version>v[0-9]+)?\.?(?P<c_type>[a-z-_]+)?.(?P<c_num>[0-9]+)')
+RETRIES = 3
 
 
 class UHTTPConnection(httplib.HTTPConnection):
@@ -133,14 +134,21 @@ def _create_container(self, name, image, command, unit, **kwargs):
             tagset = ' '.join(['"{}={}"'.format(k, v) for k, v in tags.items()])
             unit.append({"section": "X-Fleet", "name": "MachineMetadata",
                          "value": tagset})
-        # post unit to fleet
-        self._put_unit(name, {"desiredState": "launched", "options": unit})
+        # post unit to fleet and retry
+        for attempt in range(RETRIES):
+            try:
+                self._put_unit(name, {"desiredState": "launched", "options": unit})
+                break
+            except:
+                if attempt == (RETRIES - 1):  # account for 0 indexing
+                    raise
 
     def start(self, name):
         """Start a container"""
         self._wait_for_container(name)
 
     def _wait_for_container(self, name):
+        failures = 0
         # we bump to 20 minutes here to match the timeout on the router and in the app unit files
         for _ in range(1200):
             states = self._get_state(name)
@@ -150,10 +158,13 @@ def _wait_for_container(self, name):
                 if subState == 'running' or subState == 'exited':
                     break
                 elif subState == 'failed':
-                    raise RuntimeError('container failed to start')
+                    # FIXME: fleet unit state reports failed when containers are fine
+                    failures += 1
+                    if failures == 10:
+                        raise RuntimeError('container failed to start')
             time.sleep(1)
         else:
-            raise RuntimeError('container failed to start')
+            raise RuntimeError('container timeout on start')
 
     def _wait_for_destroy(self, name):
         for _ in range(30):
@@ -178,7 +189,13 @@ def destroy(self, name):
         self._wait_for_destroy(name)
 
     def _destroy_container(self, name):
-        return self._delete_unit(name)
+        for attempt in range(RETRIES):
+            try:
+                self._delete_unit(name)
+                break
+            except:
+                if attempt == (RETRIES - 1):  # account for 0 indexing
+                    raise
 
     def run(self, name, image, entrypoint, command):  # noqa
         """Run a one-off command"""