move cloud infrastructure fields from formation to layer

Author: Gabriel Monroy

api/models.py

@@ -241,15 +241,8 @@ class Formation(UuidAuditedModel):
 
     owner = models.ForeignKey(settings.AUTH_USER_MODEL)
     id = models.SlugField(max_length=64)
-    flavor = models.ForeignKey('Flavor')
     layers = fields.JSONField(default='{}', blank=True)
     containers = fields.JSONField(default='{}', blank=True)
-
-    environment = models.CharField(max_length=64, default='_default')
-    
-    ssh_username = models.CharField(max_length=64, default='ubuntu')
-    ssh_private_key = models.TextField()
-    ssh_public_key = models.TextField()
 
     class Meta:
         unique_together = (('owner', 'id'),)
@@ -381,12 +374,6 @@ def _balance_containers(self, **kwargs):
 
     def __str__(self):
         return self.id
-
-    def prepare_provider(self, *args, **kwargs):
-        tasks = import_tasks(self.flavor.provider.type)
-        args = (self.id, self.flavor.provider.creds.copy(),
-                self.flavor.params.copy())
-        return tasks.prepare_formation.subtask(args)
 
     def calculate(self):
         "Return a Chef data bag item for this formation"
@@ -435,19 +422,14 @@ def converge(self, databag):
         return databag
 
     def destroy(self):
-        tasks = import_tasks(self.flavor.provider.type)
         subtasks = []
         # call a celery task to update the formation data bag
         if settings.CHEF_ENABLED:
             subtasks.extend([controller.destroy_formation.s(self.id)])  # @UndefinedVariable
         # create subtasks to terminate all nodes in parallel
-        subtasks.extend([ node.terminate() for node in self.node_set.all() ])
+        subtasks.extend([ layer.destroy() for layer in self.layer_set.all() ])
         job = group(*subtasks)
         job.apply_async().join() # block for termination
-        # purge other hosting provider infrastructure
-        tasks.cleanup_formation.delay(self.id,
-            self.flavor.provider.creds.copy(),
-            self.flavor.params.copy()).wait()
 
 
 @python_2_unicode_compatible
@@ -465,13 +447,36 @@ class Layer(UuidAuditedModel):
     formation = models.ForeignKey('Formation')
     flavor = models.ForeignKey('Flavor')
     nodes = models.PositiveSmallIntegerField(default=0)
-
-    run_list = models.CharField(max_length=512)
+    # chef settings
+    run_list = models.CharField(max_length=512)    
     initial_attributes = fields.JSONField(default='{}', blank=True)
-
+    environment = models.CharField(max_length=64, default='_default')
+    # ssh settings
+    ssh_username = models.CharField(max_length=64, default='ubuntu')
+    ssh_private_key = models.TextField()
+    ssh_public_key = models.TextField()
+    
     def __str__(self):
         return self.id
 
+    def build(self, *args, **kwargs):
+        tasks = import_tasks(self.flavor.provider.type)
+        args = (self.id, self.flavor.provider.creds.copy(),
+                self.flavor.params.copy())
+        return tasks.build_layer.subtask(args)
+
+    def destroy(self):
+        tasks = import_tasks(self.flavor.provider.type)
+        subtasks = []
+        # create subtasks to terminate all nodes in parallel
+        subtasks.extend([ node.terminate() for node in self.node_set.all() ])
+        job = group(*subtasks)
+        job.apply_async().join() # block for termination
+        # purge other hosting provider infrastructure
+        tasks.destroy_layer.delay(self.id,
+            self.flavor.provider.creds.copy(),
+            self.flavor.params.copy()).wait() # block
+
 
 @python_2_unicode_compatible
 class NodeManager(models.Manager):
@@ -518,16 +523,16 @@ def __str__(self):
         return self.id
 
     def launch(self, *args, **kwargs):
-        tasks = import_tasks(self.formation.flavor.provider.type)
+        tasks = import_tasks(self.layer.flavor.provider.type)
         args = self._prepare_launch_args()
         return tasks.launch_node.subtask(args)
 
     def _prepare_launch_args(self):
-        creds = self.formation.flavor.provider.creds.copy()
-        params = self.formation.flavor.params.copy()
+        creds = self.layer.flavor.provider.creds.copy()
+        params = self.layer.flavor.params.copy()
         params['formation'] = self.formation.id
         params['id'] = self.id
-        init = self.formation.flavor.init.copy()
+        init = self.layer.flavor.init.copy()
         if settings.CHEF_ENABLED:
             chef = init['chef'] = {}
             chef['ruby_version'] = settings.CHEF_RUBY_VERSION
@@ -543,36 +548,36 @@ def _prepare_launch_args(self):
                 chef['initial_attributes'] = self.layer.initial_attributes
         # add the formation's ssh pubkey
         init.setdefault('ssh_authorized_keys', []).append(
-                                self.formation.ssh_public_key)
+                                self.layer.ssh_public_key)
         # add all of the owner's SSH keys
         init['ssh_authorized_keys'].extend([k.public for k in self.formation.owner.key_set.all() ])
-        ssh_username = self.formation.ssh_username
-        ssh_private_key = self.formation.ssh_private_key
+        ssh_username = self.layer.ssh_username
+        ssh_private_key = self.layer.ssh_private_key
         args = (self.uuid, creds, params, init, ssh_username, ssh_private_key)
         return args
 
     def converge(self, *args, **kwargs):
-        tasks = import_tasks(self.formation.flavor.provider.type)
+        tasks = import_tasks(self.layer.flavor.provider.type)
         args = self._prepare_converge_args()
         # TODO: figure out how to store task return values in model
         return tasks.converge_node.subtask(args)
 
     def _prepare_converge_args(self):
-        ssh_username = self.formation.ssh_username
+        ssh_username = self.layer.ssh_username
         fqdn = self.fqdn
-        ssh_private_key = self.formation.ssh_private_key
+        ssh_private_key = self.layer.ssh_private_key
         args = (self.uuid, ssh_username, fqdn, ssh_private_key)
         return args
 
     def terminate(self, *args, **kwargs):
-        tasks = import_tasks(self.formation.flavor.provider.type)
+        tasks = import_tasks(self.layer.flavor.provider.type)
         args = self._prepare_terminate_args()
         # TODO: figure out how to store task return values in model
         return tasks.terminate_node.subtask(args)
 
     def _prepare_terminate_args(self):
-        creds = self.formation.flavor.provider.creds.copy()
-        params = self.formation.flavor.params.copy()
+        creds = self.layer.flavor.provider.creds.copy()
+        params = self.layer.flavor.params.copy()
         args = (self.uuid, creds, params, self.provider_id)
         return args
 

api/serializers.py

@@ -119,24 +119,14 @@ class FormationSerializer(serializers.ModelSerializer):
 
     owner = serializers.Field(source='owner.username')
     id = serializers.SlugField(default=utils.generate_app_name)
-    flavor = serializers.SlugRelatedField(slug_field='id')
-    containers = serializers.ModelField(
-        model_field=models.Formation()._meta.get_field('containers'), required=False)
+    #containers = serializers.ModelField(
+    #    model_field=models.Formation()._meta.get_field('containers'), required=False)
 
     class Meta:
         """Metadata options for a FormationSerializer."""
         model = models.Formation
         read_only_fields = ('created', 'updated')
 
-    @property
-    def data(self):
-        "Custom data property that removes secure fields"
-        d = super(FormationSerializer, self).data
-        for f in ('ssh_private_key',):
-            if f in d:
-                del d[f]
-        return d
-
 
 class LayerSerializer(serializers.ModelSerializer):
 
@@ -151,6 +141,15 @@ class Meta:
         model = models.Layer
         read_only_fields = ('created', 'updated')
 
+    @property
+    def data(self):
+        "Custom data property that removes secure fields"
+        d = super(LayerSerializer, self).data
+        for f in ('ssh_private_key',):
+            if f in d:
+                del d[f]
+        return d
+
 
 class NodeSerializer(serializers.ModelSerializer):
 

api/tests/build.py

@@ -40,7 +40,7 @@ def test_build(self):
         can post new builds to a formation
         """
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']

api/tests/config.py

@@ -37,7 +37,7 @@ def test_config(self):
         new versions can be created using a PATCH
         """
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']

api/tests/container.py

@@ -36,12 +36,12 @@ def setUp(self):
 
     def test_container_scale(self):
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']
         url = '/api/formations/{formation_id}/layers'.format(**locals())
-        body = {'id': 'runtime', 'run_list': 'recipe[deis::runtime]'}
+        body = {'id': 'runtime', 'flavor': 'autotest', 'run_list': 'recipe[deis::runtime]'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         # scale runtime layer up
@@ -87,12 +87,12 @@ def test_container_scale(self):
 
     def test_container_balance(self):
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']
         url = '/api/formations/{formation_id}/layers'.format(**locals())
-        body = {'id': 'runtime', 'run_list': 'recipe[deis::runtime]'}
+        body = {'id': 'runtime', 'flavor': 'autotest', 'run_list': 'recipe[deis::runtime]'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         # scale layer

api/tests/formation.py

@@ -9,7 +9,6 @@
 import json
 
 from django.test import TestCase
-from Crypto.PublicKey import RSA
 
 
 class FormationTest(TestCase):
@@ -37,11 +36,10 @@ def test_formation(self):
         Test that a user can create, read, update and delete a node formation
         """
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']
-        self.assertIn('flavor', response.data)
         self.assertIn('layers', response.data)
         self.assertIn('containers', response.data)
         response = self.client.get('/api/formations')
@@ -57,39 +55,26 @@ def test_formation(self):
         self.assertEqual(response.status_code, 204)
 
     def test_formation_auto_id(self):
-        body = {'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post('/api/formations', json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         self.assertTrue(response.data['id'])
         return response
-
-    def test_formation_ssh_override(self):
-        key = RSA.generate(2048)
-        body = {'id': 'autotest', 'flavor': 'autotest', 'image': 'deis/autotest',
-                'ssh_private_key': key.exportKey('PEM'), 
-                'ssh_public_key': key.exportKey('OpenSSH')}
-        url = '/api/formations'
-        response = self.client.post(url, json.dumps(body), content_type='application/json')
-        self.assertEqual(response.status_code, 201)
-        self.assertIn('ssh_public_key', response.data)
-        self.assertEquals(response.data['ssh_public_key'], body['ssh_public_key'])
-        # ssh private key should be hidden
-        self.assertNotIn('ssh_private_key', response.data)
 
     def test_formation_errors(self):
         # test duplicate id
-        body = {'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {}
         response = self.client.post('/api/formations', json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         self.assertTrue(response.data['id'])
-        body = {'id': response.data['id'], 'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': response.data['id']}
         response = self.client.post('/api/formations', json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 400)
         self.assertEqual(json.loads(response.content), 'Formation with this Id already exists.')
 
     def test_formation_scale_errors(self):
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']
@@ -101,7 +86,7 @@ def test_formation_scale_errors(self):
         self.assertEqual(json.loads(response.content), 'Must create a "runtime" layer to host containers')
         # scaling containers without any runtime nodes should throw an error
         url = '/api/formations/{formation_id}/layers'.format(**locals())
-        body = {'id': 'runtime', 'run_list': 'recipe[deis::runtime]'}
+        body = {'id': 'runtime', 'flavor': 'autotest', 'run_list': 'recipe[deis::runtime]'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         url = '/api/formations/{formation_id}/scale/containers'.format(**locals())
@@ -112,7 +97,7 @@ def test_formation_scale_errors(self):
 
     def test_formation_actions(self):
         url = '/api/formations'
-        body = {'id': 'autotest', 'flavor': 'autotest', 'image': 'deis/autotest'}
+        body = {'id': 'autotest'}
         response = self.client.post(url, json.dumps(body), content_type='application/json')
         self.assertEqual(response.status_code, 201)
         formation_id = response.data['id']