ref(scheduler): rework the way deis run is handled, timeouts and more

Use the same "is the pod ready" function as normal deployments, also give deis run 20 mins max (arbitrary but is somewhat what the gunicorn worker can do)
This also the API to return exit_code instead of the cryptic rc value

On failure the whole Pod manifest is no longer returned. This change affects how we do tests here but minimally

Mocking was improved to help move a Pod from Pending to Running and if it is a deis run Pod (not attached to RC) then it also goes to Succeeded state. This helps with test coverage but is not the full Pod state transition route

Author: helgi

rootfs/api/models/app.py

@@ -123,6 +123,24 @@ def _get_command(self, container_type):
             # handle special case for Dockerfile deployments
             return '' if container_type == 'cmd' else 'start {}'.format(container_type)
 
+    def _get_command_run(self, command):
+        # SECURITY: shell-escape user input
+        command = command.replace("'", "'\\''")
+
+        # if this is a procfile-based app, switch the entrypoint to slugrunner's default
+        # FIXME: remove slugrunner's hardcoded entrypoint
+        release = self.release_set.latest()
+        if release.build.procfile and \
+           release.build.sha and not \
+           release.build.dockerfile:
+            entrypoint = '/runner/init'
+            command = "'{}'".format(command)
+        else:
+            entrypoint = '/bin/bash'
+            command = "-c '{}'".format(command)
+
+        return entrypoint, command
+
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -555,35 +573,22 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             raise DeisException('No build associated with this release to run this command')
 
         # TODO: add support for interactive shell
-        # SECURITY: shell-escape user input
-        command = command.replace("'", "'\\''")
-
-        # if this is a procfile-based app, switch the entrypoint to slugrunner's default
-        # FIXME: remove slugrunner's hardcoded entrypoint
-        if release.build.procfile and \
-           release.build.sha and not \
-           release.build.dockerfile:
-            entrypoint = '/runner/init'
-            command = "'{}'".format(command)
-        else:
-            entrypoint = '/bin/bash'
-            command = "-c '{}'".format(command)
+        entrypoint, command = self._get_command_run(command)
 
         name = self._get_job_id('run') + '-' + pod_name()
-
-        msg = "{} on {} runs '{}'".format(user.username, name, command)
-        log_event(self, msg)
+        log_event(self, "{} on {} runs '{}'".format(user.username, name, command))
 
         kwargs = {
             'memory': release.config.memory,
             'cpu': release.config.cpu,
             'tags': release.config.tags,
             'envs': release.config.values,
             'version': "v{}".format(release.version),
+            'build_type': release.build.type,
         }
 
         try:
-            rc, output = self._scheduler.run(
+            exit_code, output = self._scheduler.run(
                 self.id,
                 name,
                 release.image,
@@ -592,7 +597,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
                 **kwargs
             )
 
-            return rc, output
+            return exit_code, output
         except Exception as e:
             err = '{} (run): {}'.format(name, e)
             log_event(self, err, logging.ERROR)
@@ -615,7 +620,7 @@ def list_pods(self, *args, **kwargs):
                 if p['metadata']['labels']['type'] == 'run':
                     continue
 
-                state = self._scheduler.resolve_state(p)
+                state = self._scheduler._pod_state(p).name
 
                 # follows kubelete convention - these are hidden unless show-all is set
                 if state in ['down', 'crashed']:

rootfs/api/tests/test_pods.py

@@ -578,7 +578,8 @@ def test_run_command_good(self, mock_requests):
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 200, response.data)
-        entrypoint = json.loads(response.data['output'])['spec']['containers'][0]['command'][0]
+        data = App.objects.get(id=app_id)
+        entrypoint, _ = data._get_command_run('echo hi')
         self.assertEqual(entrypoint, '/bin/bash')
 
         # docker image workflow
@@ -589,7 +590,8 @@ def test_run_command_good(self, mock_requests):
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 200, response.data)
-        entrypoint = json.loads(response.data['output'])['spec']['containers'][0]['command'][0]
+        data = App.objects.get(id=app_id)
+        entrypoint, _ = data._get_command_run('echo hi')
         self.assertEqual(entrypoint, '/bin/bash')
 
         # procfile workflow
@@ -599,12 +601,13 @@ def test_run_command_good(self, mock_requests):
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 200, response.data)
-        entrypoint = json.loads(response.data['output'])['spec']['containers'][0]['command'][0]
+        data = App.objects.get(id=app_id)
+        entrypoint, _ = data._get_command_run('echo hi')
         self.assertEqual(entrypoint, '/runner/init')
 
     def test_run_not_fail_on_debug(self, mock_requests):
         """
-        do a run with DEBUG on - https://github.com/deis/controller/issues/583
+        do a run with DEIS_DEBUG on - https://github.com/deis/controller/issues/583
         """
         env = EnvironmentVarGuard()
         env['DEIS_DEBUG'] = 'true'
@@ -642,7 +645,8 @@ def test_run_not_fail_on_debug(self, mock_requests):
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 200, response.data)
-        entrypoint = json.loads(response.data['output'])['spec']['containers'][0]['command'][0]
+        data = App.objects.get(id=app_id)
+        entrypoint, _ = data._get_command_run('echo hi')
         self.assertEqual(entrypoint, '/bin/bash')
 
     def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):

rootfs/api/tests/test_scheduler_states.py

@@ -0,0 +1,30 @@
+import unittest
+from scheduler.states import PodState
+
+
+class TestSchedulerStates(unittest.TestCase):
+    """Test Scheduler States OrderedEnum"""
+
+    def test_gt_comparison(self):
+        self.assertTrue(PodState.up > PodState.starting)
+        self.assertFalse(PodState.starting > PodState.up)
+        with self.assertRaises(TypeError):
+            self.assertTrue(PodState.up > 'starting')
+
+    def test_ge_comparison(self):
+        self.assertTrue(PodState.up >= PodState.starting)
+        self.assertFalse(PodState.starting >= PodState.up)
+        with self.assertRaises(TypeError):
+            self.assertTrue(PodState.up >= 'starting')
+
+    def test_lt_comparison(self):
+        self.assertFalse(PodState.up < PodState.starting)
+        self.assertTrue(PodState.starting < PodState.up)
+        with self.assertRaises(TypeError):
+            self.assertTrue(PodState.up < 'crashed')
+
+    def test_le_comparison(self):
+        self.assertFalse(PodState.up <= PodState.starting)
+        self.assertTrue(PodState.starting <= PodState.up)
+        with self.assertRaises(TypeError):
+            self.assertTrue(PodState.up <= 'crashed')

rootfs/deis/testing.py

@@ -2,6 +2,15 @@
 import string
 from deis.settings import *  # noqa
 
+# A boolean that turns on/off debug mode.
+# https://docs.djangoproject.com/en/1.9/ref/settings/#debug
+DEBUG = True
+
+# If set to True, Django's normal exception handling of view functions
+# will be suppressed, and exceptions will propagate upwards
+# https://docs.djangoproject.com/en/1.9/ref/settings/#debug-propagate-exceptions
+DEBUG_PROPAGATE_EXCEPTIONS = True
+
 # scheduler for testing
 SCHEDULER_MODULE = 'scheduler.mock'
 SCHEDULER_URL = 'http://test-scheduler.example.com'