add test coverage for cm synchronization using mock CM module

wip on task reorg

Author: Gabriel Monroy

api/models.py

@@ -15,6 +15,7 @@
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.db import models
+from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
 from django.dispatch.dispatcher import Signal
 from django.utils.encoding import python_2_unicode_compatible
@@ -435,22 +436,21 @@ def calculate(self):
         """Calculate and update the application databag"""
         d = {}
         d['id'] = self.id
-        release = self.release_set.all().order_by('-created')[0]
         d['release'] = {}
-        d['release']['version'] = release.version
-        d['release']['config'] = release.config.values
-        d['release']['image'] = release.image
-        d['release']['build'] = {}
-        if release.build:
-            d['release']['build']['url'] = release.build.url
-            d['release']['build']['procfile'] = release.build.procfile
-        # add collaborators TODO: add sharing
+        releases = self.release_set.all().order_by('-created')
+        if releases:
+            release = releases[0]
+            d['release']['version'] = release.version
+            d['release']['config'] = release.config.values
+            d['release']['image'] = release.image
+            d['release']['build'] = {}
+            if release.build:
+                d['release']['build']['url'] = release.build.url
+                d['release']['build']['procfile'] = release.build.procfile
+        # TODO: add proper sharing and access controls
         d['users'] = {}
         for u in (self.owner.username,):
             d['users'][u] = 'admin'
-#         # call a celery task to update the data bag
-#         if settings.CHEF_ENABLED:
-#             controller.update_application.delay(self.id, d).wait()  # @UndefinedVariable
         return d
 
 
@@ -663,9 +663,8 @@ def push(cls, push):
         # recalculate the formation databag including the new
         # build and release
         databag = formation.calculate()
-        # if enabled, force-converge all of the chef nodes
-        if settings.CONVERGE_ON_PUSH is True:
-            formation.converge(databag)
+        # force-converge all of the chef nodes
+        formation.converge(databag)
         # return the databag object so the git-receive hook
         # can tell the user about proxy URLs, etc.
         return databag
@@ -731,5 +730,57 @@ def new_release(sender, **kwargs):
     return release
 
 
+def calculate(self):
+    """
+    Calculate configuration management representation
+    for this user account
+    """
+    data = {'id': self.username, 'ssh_keys': {}}
+    for k in self.key_set.all():
+        data['ssh_keys'][k.id] = k.public
+    return data
+
+# attach to built-in django user
+User.calculate = calculate
+
+# define update/delete callbacks for synchronizing
+# models with the configuration management backend
+
+
+def update_user(sender, **kwargs):
+    user = kwargs['instance']
+    tasks.publish_user.delay(user.username, user.calculate()).wait()
+
+
+def update_key(sender, **kwargs):
+    user = kwargs['instance'].owner
+    tasks.publish_user.delay(user.username, user.calculate()).wait()
+
+
+def update_app(sender, **kwargs):
+    tasks.publish_app.delay(kwargs['instance'].id).wait()
+
+
+def delete_app(sender, **kwargs):
+    tasks.purge_app.delay(kwargs['instance'].id).wait()
+
+
+def update_formation(sender, **kwargs):
+    tasks.publish_formation.delay(kwargs['instance'].id).wait()
+
+
+def delete_formation(sender, **kwargs):
+    tasks.purge_formation.delay(kwargs['instance'].id).wait()
+
+# use django signals to synchronize database updates with
+# the configuration management backend
+post_save.connect(update_user, sender=User)
+post_save.connect(update_key, sender=Key)
+post_delete.connect(update_key, sender=Key)
+post_save.connect(update_app, sender=App)
+post_delete.connect(delete_app, sender=App)
+post_save.connect(update_formation, sender=Formation)
+post_delete.connect(delete_formation, sender=Formation)
+
 # import tasks after models are defined
 from api import tasks

api/tasks.py

@@ -86,8 +86,13 @@ def converge_node(node_id):
 @task
 def run_node(app_id, command):
     app = App.objects.get(id=app_id)
-    node = app.node_set.order_by('?')[0]
-    release = app.release_set.order_by('-created')[0]
+    nodes = app.formation.node_set.order_by('?')
+    releases = app.release_set.order_by('-created')
+    if not nodes:
+        raise EnvironmentError('No nodes available to run command')
+    if not releases:
+        raise EnvironmentError('No release to run command against')
+    node, release = nodes[0], releases[0]
     # prepare ssh command
     version = release.version
     docker_args = ' '.join(
@@ -125,3 +130,40 @@ def destroy_formation(formation_id):
     group([destroy_layer.s(l.id) for l in formation.layer_set.all()]).apply_async().join()
     formation.delete()
     return formation_id
+
+
+@task
+def converge_controller():
+    CM.converge_controller()
+
+
+@task
+def publish_user(username, data):
+    CM.publish_user(username, data)
+    converge_controller.delay().wait()
+    return username
+
+
+@task
+def publish_app(app_id):
+    app = App.objects.get(id=app_id)
+    data = app.calculate()
+    CM.publish_app(app_id, data)
+    converge_controller.delay().wait()
+    return app_id
+
+
+@task
+def purge_app(app_id):
+    CM.purge_app(app_id)
+    converge_controller.delay().wait()
+    return app_id
+
+
+@task
+def publish_formation(formation_id, delete=False):
+    if delete is True:
+        return CM.purge_formation(formation_id)
+    formation = Formation.objects.get(id=formation_id)
+    data = formation.calculate()
+    return CM.publish_formation(formation_id, data)

api/tests/app.py

@@ -38,12 +38,12 @@ def setUp(self):
         self.assertEqual(response.status_code, 201)
         # create & scale a basic formation
         url = '/api/formations/{formation_id}/layers'.format(**locals())
-        body = {'id': 'proxy', 'flavor': 'autotest', 'type': 'proxy',
+        body = {'id': 'proxy', 'flavor': 'autotest', 'proxy': True,
                 'run_list': 'recipe[deis::proxy]'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         url = '/api/formations/{formation_id}/layers'.format(**locals())
-        body = {'id': 'runtime', 'flavor': 'autotest', 'type': 'runtime',
+        body = {'id': 'runtime', 'flavor': 'autotest', 'runtime': True,
                 'run_list': 'recipe[deis::proxy]'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
@@ -75,6 +75,25 @@ def test_app(self):
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204)
 
+    def test_app_cm(self):
+        """
+        Test that configuration management is updated on app changes
+        """
+        url = '/api/apps'
+        body = {'formation': 'autotest'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        app_id = response.data['id']  # noqa
+        path = os.path.join(settings.TEMPDIR, 'app-{}'.format(app_id))
+        with open(path) as f:
+            data = json.loads(f.read())
+        self.assertIn('id', data)
+        self.assertEquals(data['id'], app_id)
+        url = '/api/apps/{app_id}'.format(**locals())
+        response = self.client.delete(url)
+        self.assertEqual(response.status_code, 204)
+        self.assertFalse(os.path.exists(path))
+
     def test_app_override_id(self):
         body = {'formation': 'autotest', 'id': 'myid'}
         response = self.client.post('/api/apps', json.dumps(body),

api/tests/formation.py

@@ -7,9 +7,12 @@
 from __future__ import unicode_literals
 
 import json
+import os.path
 
 from django.test import TestCase
 
+from deis import settings
+
 
 class FormationTest(TestCase):
 
@@ -53,6 +56,25 @@ def test_formation(self):
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204)
 
+    def test_formation_cm(self):
+        """
+        Test that configuration management is updated on formation changes
+        """
+        url = '/api/formations'
+        body = {'id': 'autotest'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        formation_id = response.data['id']  # noqa
+        path = os.path.join(settings.TEMPDIR, 'formation-{}'.format(formation_id))
+        with open(path) as f:
+            data = json.loads(f.read())
+        self.assertIn('id', data)
+        self.assertEquals(data['id'], formation_id)
+        url = '/api/formations/{formation_id}'.format(**locals())
+        response = self.client.delete(url)
+        self.assertEqual(response.status_code, 204)
+        self.assertFalse(os.path.exists(path))
+
     def test_formation_id(self):
         body = {'id': 'autotest'}
         response = self.client.post('/api/formations', json.dumps(body),

api/tests/key.py

@@ -7,9 +7,12 @@
 from __future__ import unicode_literals
 
 import json
+import os.path
 
 from django.test import TestCase
 
+from deis import settings
+
 
 class KeyTest(TestCase):
 
@@ -41,6 +44,29 @@ def test_key(self):
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204)
 
+    def test_key_cm(self):
+        """
+        Test that creating and deleting a key updates configuration management
+        """
+        url = '/api/keys'
+        body = {'id': 'mykey@box.local', 'public': 'ssh-rsa XXX'}
+        response = self.client.post(url, json.dumps(body), content_type='application/json')
+        self.assertEqual(response.status_code, 201)
+        key_id = response.data['id']
+        path = os.path.join(settings.TEMPDIR, 'user-autotest')
+        with open(path) as f:
+            data = json.loads(f.read())
+        self.assertIn('id', data)
+        self.assertEquals(data['id'], 'autotest')
+        self.assertIn(body['id'], data['ssh_keys'])
+        self.assertEqual(body['public'], data['ssh_keys'][body['id']])
+        url = '/api/keys/{key_id}'.format(**locals())
+        response = self.client.delete(url)
+        self.assertEqual(response.status_code, 204)
+        with open(path) as f:
+            data = json.loads(f.read())
+        self.assertNotIn(body['id'], data['ssh_keys'])
+
     def test_key_duplicate(self):
         """
         Test that a user cannot add a duplicate key