move run_node to CM module, add container list/info endpoints, beef test coverage

Author: Gabriel Monroy

api/models.py

@@ -497,12 +497,10 @@ def logs(self):
     def run(self, command):
         """Run a one-off command in an ephemeral app container."""
         nodes = self.formation.node_set.order_by('?')
-        releases = self.release_set.order_by('-created')
         if not nodes:
             raise EnvironmentError('No nodes available to run command')
-        if not releases:
-            raise EnvironmentError('No release available to run command')
-        node, release = nodes[0], releases[0]
+        app_id, node = self.id, nodes[0]
+        release = self.release_set.order_by('-created')[0]
         # prepare ssh command
         version = release.version
         docker_args = ' '.join(
@@ -513,7 +511,7 @@ def run(self, command):
                    "`find /app/.profile.d/*.sh -type f`; do . $profile; done"
         command = "/bin/sh -c '{base_cmd} && {command}'".format(**locals())
         command = "sudo docker run {docker_args} {command}".format(**locals())
-        return node.run(self, command)
+        return node.run(command)
 
 
 @python_2_unicode_compatible

api/serializers.py

@@ -139,15 +139,6 @@ class Meta:
         model = models.Layer
         read_only_fields = ('created', 'updated')
 
-    @property
-    def data(self):
-        """Custom data property that removes secure fields"""
-        d = super(LayerSerializer, self).data
-        for f in []:  # ('ssh_private_key',):
-            if f in d:
-                del d[f]
-        return d
-
 
 class NodeSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.Node` model."""

api/tasks.py

@@ -5,11 +5,9 @@
 from celery import task
 from celery.canvas import group
 
-from api.ssh import connect_ssh, exec_ssh
 from deis import settings
 from provider import import_provider_module
 
-# now that we've defined models that may be imported by celery tasks
 # import user-defined config management module
 CM = importlib.import_module(settings.CM_MODULE)
 
@@ -54,10 +52,7 @@ def converge_node(node):
 
 @task
 def run_node(node, command):
-    ssh = connect_ssh(node.layer.ssh_username,
-                      node.fqdn, 22,
-                      node.layer.ssh_private_key)
-    output, rc = exec_ssh(ssh, command, pty=True)
+    output, rc = CM.run_node(node.flat(), command)
     return output, rc
 
 

api/tests/__init__.py

@@ -27,6 +27,7 @@ def send_patch(self, path, data='', content_type='application/octet-stream',
 RequestFactory.patch = construct_patch
 Client.patch = send_patch
 
+from .app import *  # noqa
 from .auth import *  # noqa
 from .build import *  # noqa
 from .config import *  # noqa

api/tests/app.py

@@ -117,10 +117,8 @@ def test_app_actions(self):
         if not os.path.exists(settings.DEIS_LOG_DIR):
             os.mkdir(settings.DEIS_LOG_DIR)
         path = os.path.join(settings.DEIS_LOG_DIR, app_id + '.log')
-        try:
-            os.remove(path)  # cleanup any old log files
-        except:
-            pass
+        if os.path.exists(path):
+            os.remove(path)
         url = '/api/apps/{app_id}/logs'.format(**locals())
         response = self.client.post(url)
         self.assertEqual(response.status_code, 404)
@@ -133,12 +131,27 @@ def test_app_actions(self):
         self.assertEqual(response.data, FAKE_LOG_DATA)
         # test run
         url = '/api/apps/{app_id}/run'.format(**locals())
-        body = {'commands': 'ls -al'}
+        body = {'command': 'ls -al'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 200)
-        self.assertIn('.gitignore', response.data[0])
         self.assertEqual(response.data[1], 0)
 
+    def test_app_errors(self):
+        formation_id, app_id = 'autotest', 'autotest-errors'
+        url = '/api/apps'
+        body = {'formation': formation_id, 'id': app_id}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        app_id = response.data['id']  # noqa
+        url = '/api/formations/{formation_id}/scale'.format(**locals())
+        body = {'proxy': 0, 'runtime': 0}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        url = '/api/apps/{app_id}/run'.format(**locals())
+        body = {'command': 'ls -al'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertContains(response, 'No nodes available to run command', status_code=400)
+
 
 FAKE_LOG_DATA = """
 2013-08-15 12:41:25 [33454] [INFO] Starting gunicorn 17.5

api/tests/container.py

@@ -65,7 +65,7 @@ def setUp(self):
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 200)
 
-    def test_container_scale(self):
+    def test_container(self):
         url = '/api/apps'
         body = {'formation': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
@@ -89,6 +89,16 @@ def test_container_scale(self):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data['containers'], json.dumps(body))
+        # test listing/retrieving container info
+        url = "/api/apps/{app_id}/containers/web".format(**locals())
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(len(response.data['results']), 4)
+        num = response.data['results'][0]['num']
+        url = "/api/apps/{app_id}/containers/web/{num}".format(**locals())
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['num'], num)
         # scale down
         url = "/api/apps/{app_id}/scale".format(**locals())
         body = {'web': 2, 'worker': 1}
@@ -115,7 +125,7 @@ def test_container_scale(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data['containers'], json.dumps(body))
 
-    def test_container_scale_errors(self):
+    def test_container_errors(self):
         url = '/api/apps'
         body = {'formation': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
@@ -126,7 +136,7 @@ def test_container_scale_errors(self):
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertContains(response, 'Invalid scaling format', status_code=400)
 
-    def test_container_scale_single_layer(self):
+    def test_container_single_layer(self):
         # create & scale a single layer formation
         response = self.client.post('/api/formations', json.dumps(
             {'id': 'single-layer', 'domain': 'localhost.localdomain'}),
@@ -191,7 +201,7 @@ def test_container_scale_single_layer(self):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.data['containers'], json.dumps(body))
 
-    def test_container_scale_allocation(self):
+    def test_container_allocation(self):
         url = '/api/apps'
         formation_id = 'autotest'
         body = {'formation': formation_id}

api/tests/node.py

@@ -165,3 +165,32 @@ def test_node_scale_errors(self):
         body = {'runtime': 1}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertContains(response, 'No provider credentials available', status_code=400)
+
+    def test_node_actions(self):
+        url = '/api/formations'
+        body = {'id': 'autotest'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        formation_id = response.data['id']
+        url = '/api/formations/{formation_id}/layers'.format(**locals())
+        body = {'id': 'runtime', 'flavor': 'autotest', 'runtime': True}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        url = '/api/formations/{formation_id}/scale'.format(**locals())
+        body = {'runtime': 1}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        # get our node
+        url = '/api/formations/{formation_id}/nodes'.format(**locals())
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 1)
+        node_id = response.data['results'][0]['id']
+        url = '/api/formations/{formation_id}/nodes/{node_id}'.format(**locals())
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(node_id, response.data['id'])
+        node = response.data
+        url = '/api/nodes/{id}/converge'.format(**node)
+        response = self.client.post(url)
+        self.assertEqual(response.status_code, 200)

api/urls.py

@@ -300,6 +300,10 @@
     url(r'^apps/(?P<id>[a-z0-9-]+)/releases/?',
         views.AppReleaseViewSet.as_view({'get': 'list'})),
     # application infrastructure
+    url(r'^apps/(?P<id>[a-z0-9-]+)/containers/(?P<type>[a-z0-9-]+)/(?P<num>[a-z0-9-]+)/?',
+        views.AppContainerViewSet.as_view({'get': 'retrieve'})),
+    url(r'^apps/(?P<id>[a-z0-9-]+)/containers/(?P<type>[a-z0-9-.]+)/?',
+        views.AppContainerViewSet.as_view({'get': 'list'})),
     url(r'^apps/(?P<id>[a-z0-9-]+)/containers/?',
         views.AppContainerViewSet.as_view({'get': 'list'})),
     # application actions
@@ -315,6 +319,8 @@
     url(r'^apps/?',
         views.AppViewSet.as_view({'post': 'create', 'get': 'list'})),
     # nodes
+    url(r'^nodes/(?P<node>[a-z0-9-]+)/converge/?',
+        views.NodeViewSet.as_view({'post': 'converge'})),
     url(r'^nodes/(?P<node>[a-z0-9-]+)/?',
         views.NodeViewSet.as_view({
             'get': 'retrieve', 'delete': 'destroy'})),

api/views.py

@@ -242,6 +242,11 @@ class NodeViewSet(FormationNodeViewSet):
     def get_queryset(self, **kwargs):
         return self.model.objects.filter(owner=self.request.user)
 
+    def converge(self, request, **kwargs):
+        node = self.get_object()
+        output, _ = node.converge()
+        return Response(output, status=status.HTTP_200_OK, content_type='text/plain')
+
 
 class AppViewSet(OwnerViewSet):
     """RESTful views for :class:`~api.models.App`."""
@@ -288,7 +293,11 @@ def logs(self, request, **kwargs):
 
     def run(self, request, **kwargs):
         app = self.get_object()
-        output_and_rc = app.run(request.DATA['commands'])
+        command = request.DATA['command']
+        try:
+            output_and_rc = app.run(command)
+        except EnvironmentError as e:
+            return Response(str(e), status=status.HTTP_400_BAD_REQUEST)
         return Response(output_and_rc, status=status.HTTP_200_OK,
                         content_type='text/plain')
 
@@ -376,9 +385,13 @@ class AppContainerViewSet(OwnerViewSet):
     def get_queryset(self, **kwargs):
         app = models.App.objects.get(
             owner=self.request.user, id=self.kwargs['id'])
-        return self.model.objects.filter(owner=self.request.user, app=app)
+        qs = self.model.objects.filter(owner=self.request.user, app=app)
+        container_type = self.kwargs.get('type')
+        if container_type:
+            qs = qs.filter(type=container_type)
+        return qs
 
     def get_object(self, *args, **kwargs):
         qs = self.get_queryset(**kwargs)
-        obj = qs.get(pk=self.kwargs['container'])
+        obj = qs.get(num=self.kwargs['num'])
         return obj

cm/chef.py

@@ -137,6 +137,14 @@ def converge_node(node):
     return output, rc
 
 
+def run_node(node, command):
+    ssh = connect_ssh(node.layer.ssh_username,
+                      node.fqdn, 22,
+                      node.layer.ssh_private_key)
+    output, rc = exec_ssh(ssh, command, pty=True)
+    return output, rc
+
+
 def converge_formation(formation):
     nodes = formation.node_set.all()
     subtasks = []