chore(release): add autodeploy flag

duanhongyi · duanhongyi · commit f3ba668f2102 · 2024-08-27T11:11:47.000+08:00
diff --git a/rootfs/api/migrations/0012_appsettings_autodeploy.py b/rootfs/api/migrations/0012_appsettings_autodeploy.py
@@ -0,0 +1,27 @@
+# Generated by Django 4.2.15 on 2024-08-27 02:18
+
+from django.db import migrations, models
+
+
+def migration_dryccfile(apps, schema_editor):
+    build_model = apps.get_model('api', 'Build')
+    for build in build_model.objects.all():
+        if build.dryccfile and isinstance(build.dryccfile.get('run'), dict):
+            build.dryccfile['run'] = [build.dryccfile['run']]
+            build.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_merge_20240815_0955'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appsettings',
+            name='autodeploy',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.RunPython(migration_dryccfile),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
@@ -113,6 +113,19 @@ def save(self, *args, **kwargs):
     def procfile_types(self):
         return list(self.structure.keys())
 
+    def check_procfile_types(self, procfile_types):
+        """
+        check available procfile types
+        """
+        if not procfile_types:
+            procfile_types = self.procfile_types
+        else:
+            invalid_procfile_types = procfile_types.difference(self.procfile_types)
+            if len(invalid_procfile_types) != 0:
+                raise DryccException("process type {} is not included in procfile".
+                                     format(','.join(invalid_procfile_types)))
+        return procfile_types
+
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -220,38 +233,28 @@ def scale(self, user, structure):
         app_settings = self.appsettings_set.latest()
         self._scale(user, structure, release, app_settings)
 
-    def prepare(self, release):
-        prefix = f"[pipeline] prepare {release.version_name}"
-        if release.build.dryccfile:
-            if 'run' in release.build.dryccfile and release.get_run_trigger():
-                self.log(f"{prefix} starts running pipeline.run")
-                job_name = self.run(
-                    self.owner, release.get_run_image(), command=release.get_run_command(),
-                    args=release.get_run_args(), timeout=release.get_run_timeout(),
-                    expires=release.get_run_timeout())
-                state, labels = 'initializing', {'job-name': job_name}
-                for count, state in enumerate(self.scheduler().pod.watch(
-                        self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
-                    self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
-                if state != 'down':
-                    raise DryccException(f'pipeline run state error: {state}')
-                return True
-        return False
-
-    def pipeline(self, release, force_deploy=False):
+    def pipeline(self, release, procfile_types=None, force_deploy=False):
         prefix = f"[pipeline] release {release.version_name}"
         try:
-            self.log(f"{prefix} starts running...")
-            procfile_types = release.diff_procfile_types()
-            self.prepare(release)
-            if procfile_types is None or len(procfile_types) > 0:
-                self.log(f"{prefix} starts running pipeline.deploy")
+            if release.build is not None:
+                if release.build.dryccfile:
+                    for run in release.get_runners(procfile_types):
+                        self.log(f"{prefix} starts running pipeline.run {run['image']}")
+                        job_name = self.run(
+                            self.owner, run['image'], command=run['command'],
+                            args=run['args'], timeout=run['timeout'], expires=run['timeout']
+                        )
+                        state, labels = 'initializing', {'job-name': job_name}
+                        for count, state in enumerate(self.scheduler().pod.watch(
+                                self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
+                            self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
+                        if state != 'down':
+                            raise DryccException(f'pipeline run state error: {state}')
+                self.log(f"{prefix} starts running...")
                 rollback_on_failure = self.appsettings_set.latest().autorollback
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
-            else:
-                self.log(f"{prefix} no changes, skip executing pipeline.deploy")
             release.state = "succeed"
         except Exception as e:
             release.state = "crashed"
@@ -356,14 +359,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         try:
             # create application config and build the pod manifest
             self.set_application_config(release, PROCFILE_TYPE_RUN)
-            self.scheduler().job.create(
-                self.id,
-                name,
-                image if image else release.get_run_image(),
-                command if command else release.get_run_command(),
-                args if args else release.get_run_args(),
-                **kwargs
-            )
+            self.scheduler().job.create(self.id, name, image, command, args, **kwargs)
         except Exception as e:
             err = '{} ({}): {}'.format(name, PROCFILE_TYPE_RUN, e)
             raise ServiceUnavailable(err) from e
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
@@ -20,6 +20,7 @@ class AppSettings(UuidAuditedModel):
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     routable = models.BooleanField(default=True)
+    autodeploy = models.BooleanField(default=True)
     autorollback = models.BooleanField(default=True)
     autoscale = models.JSONField(default=dict, blank=True)
     label = models.JSONField(default=dict, blank=True)
@@ -63,23 +64,14 @@ def previous(self):
             prev_app_settings = None
         return prev_app_settings
 
-    def _update_routable(self, previous_settings):
-        old = getattr(previous_settings, 'routable', None)
-        new = getattr(self, 'routable', None)
+    def _update_field(self, field, previous_settings):
+        old = getattr(previous_settings, field, None)
+        new = getattr(self, field, None)
         # if nothing changed copy the settings from previous
         if new is None and old is not None:
-            setattr(self, 'routable', old)
+            setattr(self, field, old)
         elif old != new:
-            self.summary += ["{} changed routablity from {} to {}".format(self.owner, old, new)]
-
-    def _update_autorollback(self, previous_settings):
-        old = getattr(previous_settings, 'autorollback', None)
-        new = getattr(self, 'autorollback', None)
-        # if nothing changed copy the settings from previous
-        if new is None and old is not None:
-            setattr(self, 'autorollback', old)
-        elif old != new:
-            self.summary += ["{} changed autorollback from {} to {}".format(self.owner, old, new)]
+            self.summary += ["{} changed {} from {} to {}".format(self.owner, field, old, new)]
 
     def _update_autoscale(self, previous_settings):
         data = getattr(previous_settings, 'autoscale', {}).copy()
@@ -159,11 +151,15 @@ def _update_fields(self, ignore_update_fields=None):
             previous_settings = self.app.appsettings_set.latest()
         except AppSettings.DoesNotExist:
             pass
-        update_fields = ["routable", "autorollback", "autoscale", "label"]
+        update_fields = ["routable", "autodeploy", "autorollback", "autoscale", "label"]
         try:
             for update_field in update_fields:
                 if ignore_update_fields is None or update_field not in ignore_update_fields:
-                    getattr(self, "_update_%s" % update_field)(previous_settings)
+                    method = getattr(self, "_update_%s" % update_field, None)
+                    if method:
+                        method(previous_settings)
+                    else:
+                        self._update_field(update_field, previous_settings)
         except (UnprocessableEntity, NotFound):
             raise
         except Exception as e:
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
@@ -79,7 +79,8 @@ def create_release(self, user, *args, **kwargs):
                 build=self,
                 config=latest_release.config,
             )
-            run_pipeline.delay(new_release, force_deploy=False)
+            if self.app.appsettings_set.latest().autorollback:
+                run_pipeline.delay(new_release, force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
@@ -4,7 +4,6 @@
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth import get_user_model
-from api.utils import dict_diff
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
@@ -61,43 +60,22 @@ def procfile_types(self):
     def version_name(self):
         return f'v{self.version}'
 
-    def get_run_image(self):
-        """
-        In the run phase of dryccfile
-        Return the kubernetes "container image" to be sent off to the scheduler.
-        """
-        image = self.build.dryccfile.get('run', {}).get(
-            'image', self.build.get_image('run'))
-        return self.build.get_image(image, default_image=image)
-
-    def get_run_args(self):
-        """
-        In the run phase of dryccfile
-        Return the kubernetes "container arguments" to be sent off to the scheduler.
-        """
-        return self.build.dryccfile.get('run', {}).get('args', [])
-
-    def get_run_command(self):
-        """
-        In the run phase of dryccfile
-        Return the kubernetes "container command" to be sent off to the scheduler.
-        """
-        return self.build.dryccfile.get('run', {}).get('command', [])
-
-    def get_run_timeout(self):
-        return int(self.build.dryccfile.get('run', {}).get(
-            'timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT))
-
-    def get_run_trigger(self):
-        if 'ptypes' not in self.build.dryccfile.get('run', {}).get('when', {}):
-            return True
-        procfile_types = self.diff_procfile_types()
-        if procfile_types is None:
-            return True
-        for procfile_type in procfile_types:
-            if procfile_type in self.build.dryccfile['run']['when']['ptypes']:
-                return True
-        return False
+    def get_runners(self, procfile_types):
+        results = []
+        procfile_types = self.procfile_types if not procfile_types else procfile_types
+        for run in self.build.dryccfile.get('run', []):
+            for container_type in procfile_types:
+                when_ptypes = run.get('when', {}).get('ptypes', [])
+                if not when_ptypes or container_type in when_ptypes:
+                    image = run.get('image', self.build.get_image(container_type))
+                    results.append({
+                        'image': self.build.get_image(image, default_image=image),
+                        'args': run.get('args', []),
+                        'command': run.get('command', []),
+                        'timeout': run.get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
+                    })
+                    break
+        return results
 
     def get_deploy_image(self, container_type):
         """
@@ -133,30 +111,6 @@ def get_deploy_command(self, container_type):
         return self.build.dryccfile.get(
             'deploy', {}).get(container_type, {}).get('command', [])
 
-    def diff_procfile_types(self):
-        """
-        If returning None, it indicates that all procfile_types have changed.
-        """
-        def _get_full_deploy(release):
-            deploy = {}
-            for procfile_type, value in release.build.dryccfile['deploy'].items():
-                value['image'] = release.get_deploy_image(procfile_type)
-                value['args'] = release.get_deploy_args(procfile_type)
-                value['command'] = release.get_deploy_command(procfile_type)
-                deploy[procfile_type] = value
-            return deploy
-
-        pre_release = self.previous()
-        if (pre_release and pre_release.build and
-                pre_release.build.dryccfile and self.build and self.build.dryccfile):
-            deploy = _get_full_deploy(self)
-            pre_deploy = _get_full_deploy(pre_release)
-            procfile_types = set()
-            for value in dict_diff(deploy, pre_deploy).values():
-                procfile_types = procfile_types.union(value.keys())
-            return procfile_types
-        return None
-
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -217,7 +171,7 @@ def previous(self):
             prev_release = None
         return prev_release
 
-    def rollback(self, user, version=None):
+    def rollback(self, user, procfile_types=None, version=None):
         try:
             # if no version is provided then grab version from object
             version = (self.version - 1) if version is None else int(version)
@@ -237,7 +191,7 @@ def rollback(self, user, version=None):
                 summary="{} rolled back to v{}".format(user, version),
             )
             if self.build is not None:
-                run_pipeline.delay(new_release, force_deploy=True)
+                run_pipeline.delay(new_release, procfile_types, force_deploy=True)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
diff --git a/rootfs/api/serializers/schemas/dryccfile.py b/rootfs/api/serializers/schemas/dryccfile.py
@@ -18,17 +18,20 @@
             },
         },
         "run": {
-            "type": "object",
-            "properties": {
-                "image": {"type": "string"},
-                "command": {
-                    "type": "array",
-                    "items": {"type": "string"},
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "image": {"type": "string"},
+                    "command": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                    },
+                    "args": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                    }
                 },
-                "args": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                }
             },
         },
         "deploy": {
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
@@ -62,31 +62,6 @@ def run_pipeline(release, *args, **kwargs):
         signals.request_finished.send(sender=task_id)
 
 
-@shared_task(
-    autoretry_for=(ServiceUnavailable, ),
-    retry_kwargs={'max_retries': None}
-)
-def run_deploy(release, *args, **kwargs):
-    task_id = uuid.uuid4().hex
-    signals.request_started.send(sender=task_id)
-    try:
-        if release.build is not None:
-            release.app.deploy(release, *args, **kwargs)
-        release.state = "succeed"
-        release.save()
-    except Exception as e:
-        release.state = "crashed"
-        release.failed = True
-        if release.summary:
-            release.summary += " "
-        release.summary += "{} deployed a config that failed".format(release.owner)
-        # Get the exception that has occured
-        release.exception = "error: {}".format(str(e))
-        release.save()
-    finally:
-        signals.request_finished.send(sender=task_id)
-
-
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_jitter=True,
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
@@ -235,8 +235,8 @@ def test_run_without_release_should_error(self, mock_requests):
         body = {'command': 'ls -al'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {'detail': 'No build associated with this '
-                                                   'release to run this command'})
+        self.assertEqual(
+            str(response.data["detail"]), 'no build available, please deploy a release')
 
     @mock.patch('api.models.app.App.run', _mock_run)
     @mock.patch('api.models.app.App.deploy', mock_none)
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
@@ -40,6 +40,22 @@ def test_settings_routable(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         self.assertFalse(app.appsettings_set.latest().routable)
 
+    def test_settings_autodeploy(self, mock_requests):
+        """
+        Create an application with the autorollback flag turned on or off
+        """
+        # create app, expecting autodeploy to be true
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        # Set autodeploy to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autodeploy': False}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
+
     def test_settings_autorollback(self, mock_requests):
         """
         Create an application with the autorollback flag turned on or off
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
