feat(config): deploy according to procfile_type

Author: duanhongyi

rootfs/api/models/app.py

@@ -257,7 +257,7 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
                     if state != 'down':
                         raise DryccException(f'pipeline run state error: {state}')
             self.log(f"{prefix} starts running pipeline.deploy")
-            self.deploy(release, force_deploy, rollback_on_failure)
+            self.deploy(release, None, force_deploy, rollback_on_failure)
             release.state = "succeed"
         except Exception as e:
             release.state = "crashed"
@@ -268,7 +268,7 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
         release.save()
         self.log(f"{prefix} run completed...")
 
-    def deploy(self, release, force_deploy=False, rollback_on_failure=True):
+    def deploy(self, release, structure=None, force_deploy=False, rollback_on_failure=True):
         """
         Deploy a new release to this application
 
@@ -296,11 +296,13 @@ def deploy(self, release, force_deploy=False, rollback_on_failure=True):
         volumes = self.volume_set.all()
         deploys = {}
         for scale_type, replicas in self.structure.items():
+            if structure is not None and scale_type not in structure:
+                continue
             scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
             if not release.canary or scale_type in app_settings.canaries:
                 deploys[scale_type] = self._gather_app_settings(
                     release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
-        self._deploy(deploys, prev_release, release, force_deploy, rollback_on_failure)
+        self._deploy(deploys, structure, prev_release, release, force_deploy, rollback_on_failure)
         # cleanup old release objects from kubernetes
         self.cleanup_old()
         release.cleanup_old()
@@ -610,7 +612,8 @@ def _mount(self, user, volume, release, app_settings, structure=None):
             raise ServiceUnavailable(err) from e
         self.log(f'{user.username} changed volume mount for {volume}')
 
-    def _deploy(self, deploys, prev_release, release, force_deploy, rollback_on_failure):
+    def _deploy(self, deploys, structure, prev_release,
+                release, force_deploy, rollback_on_failure):
         # Sort deploys so routable comes first
         deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
         # Check if any proc type has a Deployment in progress
@@ -644,7 +647,8 @@ def _deploy(self, deploys, prev_release, release, force_deploy, rollback_on_fail
                     # This goes in the log before the rollback starts
                     self.log(err, logging.ERROR)
                     # revert all process types to old release
-                    self.deploy(prev_release, force_deploy=True, rollback_on_failure=False)
+                    self.deploy(prev_release, structure,
+                                force_deploy=True, rollback_on_failure=False)
                     # let it bubble up
                     raise DryccException('{}\n{}'.format(err, str(e))) from e
 
@@ -898,19 +902,14 @@ def _default_structure(release):
 
     def _scheduler_filter(self, **kwargs):
         labels = {'app': self.id, 'heritage': 'drycc'}
-
-        # always supply a version, either latest or a specific one
-        if 'release' not in kwargs or kwargs['release'] is None:
-            release = self.release_set.filter(failed=False).latest()
-        else:
-            release = self.release_set.get(version=kwargs['release'])
-
-        version = "v{}".format(release.version)
-        labels.update({'version': version})
-
         if 'type' in kwargs:
             labels.update({'type': kwargs['type']})
-
+        if 'version' in kwargs:
+            if isinstance(kwargs['version'], int):
+                version = "v{}".format(kwargs['version'])
+            else:
+                version = kwargs['version']
+            labels.update({'version': version})
         return labels
 
     def _build_env_vars(self, release):

rootfs/api/models/config.py

@@ -3,6 +3,7 @@
 from django.conf import settings
 from django.db import models
 from django.contrib.auth import get_user_model
+from api.utils import dict_diff
 from api.exceptions import DryccException, UnprocessableEntity
 from .release import Release
 from .base import UuidAuditedModel
@@ -16,6 +17,9 @@ class Config(UuidAuditedModel):
     Set of configuration values applied as environment variables
     during runtime execution of the Application.
     """
+    procfile_fields = ("lifecycle_post_start", "lifecycle_pre_stop", "tags", "limits",
+                       "healthcheck", "termination_grace_period")
+    all_diff_fields = ("values", "registry") + procfile_fields
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
@@ -53,6 +57,13 @@ def previous(self):
             prev_release = None
         return prev_release
 
+    def diff(self, config=None):
+        old_config = config if config else self.previous()
+        result = {}
+        for field in self.all_diff_fields:
+            result[field] = dict_diff(getattr(self, field), getattr(old_config, field))
+        return result
+
     def save(self, **kwargs):
         """merge the old config with the new"""
         try:

rootfs/api/models/release.py

@@ -3,7 +3,6 @@
 from django.conf import settings
 from django.db import models
 from django.contrib.auth import get_user_model
-from api.utils import dict_diff
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
@@ -385,127 +384,13 @@ def save(self, *args, **kwargs):  # noqa
                 else:
                     self.summary += "{} deployed {}".format(self.build.owner, self.build.image)
             elif self.config != old_config:
-                # if env vars change, log the dict diff
-                dict1 = self.config.values
-                dict2 = old_config.values if old_config else {}
-                diff = dict_diff(dict1, dict2)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the limits changed, log the dict diff
-                changes = []
-                old_limits = old_config.limits if old_config else {}
-                diff = dict_diff(self.config.limits, old_limits)
-                if diff.get('added') or diff.get('changed') or diff.get('deleted'):
-                    changes.append('limits')
-                if changes:
-                    changes = 'changed limits for '+', '.join(changes)
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the lifecycle_post_start hooks changed, log the dict diff
-                changes = []
-                old_lifecycle_post_start = old_config.lifecycle_post_start if old_config else {}
-                diff = dict_diff(self.config.lifecycle_post_start, old_lifecycle_post_start)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added lifecycle_post_start  ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed lifecycle_post_start ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted lifecycle_post_start ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the lifecycle_pre_stop hooks changed, log the dict diff
-                changes = []
-                old_lifecycle_pre_stop = old_config.lifecycle_pre_stop if old_config else {}
-                diff = dict_diff(self.config.lifecycle_pre_stop, old_lifecycle_pre_stop)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added lifecycle_pre_stop  ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed lifecycle_pre_stop ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted lifecycle_pre_stop ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-
-                # if the timeouts changed, log the dict diff
-                changes = []
-                old_timeout = old_config.termination_grace_period if old_config else {}
-                diff = dict_diff(self.config.termination_grace_period, old_timeout)
-                if diff.get('added') or diff.get('changed') or diff.get('deleted'):
-                    changes.append('termination_grace_period')
-                if changes:
-                    changes = 'changed timeouts for '+', '.join(changes)
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the tags changed, log the dict diff
-                changes = []
-                old_tags = old_config.tags if old_config else {}
-                diff = dict_diff(self.config.tags, old_tags)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added tag ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed tag ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted tag ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the registry information changed, log the dict diff
-                changes = []
-                old_registry = old_config.registry if old_config else {}
-                diff = dict_diff(self.config.registry, old_registry)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added registry info ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed registry info ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted registry info ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the healthcheck information changed, log the dict diff
-                changes = []
-                old_healthcheck = old_config.healthcheck if old_config else {}
-                diff = dict_diff(self.config.healthcheck, old_healthcheck)
-                # try to be as succinct as possible
-                added = ', '.join(list(map(lambda x: 'default' if x == '' else x, [k for k in diff.get('added', {})])))  # noqa
-                added = 'added healthcheck info for proc type ' + added if added else ''
-                changed = ', '.join(list(map(lambda x: 'default' if x == '' else x, [k for k in diff.get('changed', {})])))  # noqa
-                changed = 'changed healthcheck info for proc type ' + changed if changed else ''
-                deleted = ', '.join(list(map(lambda x: 'default' if x == '' else x, [k for k in diff.get('deleted', {})])))  # noqa
-                deleted = 'deleted healthcheck info for proc type ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
+                for field, diff in self.config.diff(old_config).items():
+                    diff_list = []
+                    for diff_type, values in diff.items():
+                        diff_list.append(f'{diff_type} {field} {", ".join(values.keys())}')
+                    if diff_list:
+                        changes = ', '.join(diff_list)
+                        self.summary += "{} {}".format(self.config.owner, changes)
             if not self.summary:
                 if self.version == 1:
                     self.summary = "{} created the initial release".format(self.owner)

rootfs/api/tests/__init__.py

@@ -2,6 +2,7 @@
 import random
 import requests_mock
 import time
+import unittest
 from os.path import dirname, realpath
 
 from django.test.runner import DiscoverRunner
@@ -51,7 +52,7 @@ def run_tests(self, test_labels, extra_tests=None, **kwargs):
             test_labels, extra_tests, **kwargs)
 
 
-class DryccTestCase(APITestCase):
+class DryccBaseTestCase(unittest.TestCase):
 
     def create_app(self, name=None):
         body = {}
@@ -63,14 +64,23 @@ def create_app(self, name=None):
         self.assertIn('id', response.data)
         return response.data['id']
 
+    def assertPodContains(self, pods, app_id, procfile_type, version, state="up"):
+        for pod in pods:
+            if (pod["type"] == procfile_type and
+                    pod["release"] == version and pod["state"] == state):
+                pod_name = app_id + '-%s-[0-9]{1,10}-[a-z0-9]{5}' % procfile_type
+                self.assertRegex(pod['name'], pod_name)
+                return
+        raise ValueError(
+            "pod not contains: procfile_type={}, version={}, state={}".format(
+                procfile_type, version, state
+            )
+        )
 
-class DryccTransactionTestCase(APITransactionTestCase):
-    def create_app(self, name=None):
-        body = {}
-        if name:
-            body = {'id': name}
 
-        response = self.client.post('/v2/apps', body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('id', response.data)
-        return response.data['id']
+class DryccTransactionTestCase(DryccBaseTestCase, APITransactionTestCase):
+    pass
+
+
+class DryccTestCase(DryccBaseTestCase, APITestCase):
+    pass