feat(release): add release deploy

Author: duanhongyi

rootfs/api/models/app.py

@@ -21,7 +21,7 @@
 
 from api.utils import get_session
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
-from api.utils import generate_app_name, apply_tasks
+from api.utils import DeployLock, generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
 from .gateway import Gateway, Route
 from .limit import LimitPlan
@@ -255,16 +255,17 @@ def pipeline(self, release, procfile_types=None, force_deploy=False):
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
-            release.state = "succeed"
+            if release.state == "created":
+                release.state = "succeed"
+            release.add_condition(state="succeed", action="pipeline", ptypes=procfile_types)
         except Exception as e:
-            release.state = "crashed"
-            release.failed = True
-            if release.summary:
-                release.summary += " "
-            release.summary += "{} pipeline a release that failed".format(self.owner)
-            release.exception = "error: {}".format(str(e))
+            release.failed, release.state = True, "crashed"
+            release.add_condition(
+                state="crashed", action="pipeline", ptypes=procfile_types, exception=str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
-        release.save()
+        finally:
+            DeployLock(self.pk).release(procfile_types)
+            release.save(update_fields=["state", "failed"])  # avoid overwriting other fields
         self.log(f"{prefix} run completed...")
 
     def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_failure=True):

rootfs/api/models/build.py

@@ -3,7 +3,6 @@
 from django.db import models
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException, Conflict
-from api.tasks import run_pipeline
 from .base import UuidAuditedModel
 
 User = get_user_model()
@@ -79,8 +78,8 @@ def create_release(self, user, *args, **kwargs):
                 build=self,
                 config=latest_release.config,
             )
-            if self.app.appsettings_set.latest().autorollback:
-                run_pipeline.delay(new_release, force_deploy=False)
+            if self.app.appsettings_set.latest().autodeploy:
+                new_release.deploy(force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
@@ -95,7 +94,8 @@ def create_release(self, user, *args, **kwargs):
                     self.owner, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
-                new_release.save()
+                # avoid overwriting other fields
+                new_release.save(update_fields=["state", "failed", "summary", "exception"])
             if 'new_release' not in locals():
                 self.delete()
             raise DryccException(str(e)) from e

rootfs/api/models/release.py

@@ -1,13 +1,19 @@
 import logging
 
+from datetime import datetime
+from django.utils import timezone
 from django.conf import settings
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth import get_user_model
+from django.db.models import F, Func, Value, JSONField
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
+
+from ..utils import DeployLock
+
 from .base import UuidAuditedModel
 from .appsettings import AppSettings
 
@@ -34,6 +40,7 @@ class Release(UuidAuditedModel):
     summary = models.TextField(blank=True, null=True)
     failed = models.BooleanField(default=False)
     exception = models.TextField(blank=True, null=True)
+    conditions = models.JSONField(default=list)
 
     config = models.ForeignKey('Config', on_delete=models.CASCADE)
     build = models.ForeignKey('Build', null=True, on_delete=models.CASCADE)
@@ -46,10 +53,6 @@ class Meta:
     def __str__(self):
         return "{0}-{1}".format(self.app.id, self.version_name)
 
-    @property
-    def deploying(self):
-        return self.build is not None and self.state == "created"
-
     @property
     def procfile_types(self):
         if self.build is not None:
@@ -77,6 +80,18 @@ def get_runners(self, procfile_types):
                     break
         return results
 
+    def add_condition(self, **kwargs):
+        if "created" not in kwargs:
+            kwargs["created"] = datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT)
+        type(self).objects.filter(pk=self.pk).update(
+            conditions=Func(
+                F("conditions"),
+                Value(["0"]),
+                Value(kwargs, JSONField()),
+                function="jsonb_insert",
+            )
+        )
+
     def get_deploy_image(self, container_type):
         """
         In the deploy phase of dryccfile
@@ -147,6 +162,12 @@ def get_port(self, procfile_type):
             procfile_type, {}).get(
                 'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
 
+    def deploy(self, procfile_types=None, force_deploy=False):
+        lock = DeployLock(self.app.pk)
+        if not lock.acquire(procfile_types, force=force_deploy):
+            raise DryccException('there is an executing pipeline, please wait or force deploy')
+        run_pipeline.delay(self, procfile_types, force_deploy)
+
     def previous(self):
         """
         Return the previous Release to this one.
@@ -191,7 +212,7 @@ def rollback(self, user, procfile_types=None, version=None):
                 summary="{} rolled back to v{}".format(user, version),
             )
             if self.build is not None:
-                run_pipeline.delay(new_release, procfile_types, force_deploy=True)
+                new_release.deploy(procfile_types, force_deploy=True)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
@@ -206,7 +227,8 @@ def rollback(self, user, procfile_types=None, version=None):
                     self.owner)
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
-                new_release.save()
+                # avoid overwriting other fields
+                new_release.save(update_fields=["state", "failed", "summary", "exception"])
             raise DryccException(str(e)) from e
 
     def cleanup_old(self, procfile_types=None):
@@ -378,5 +400,4 @@ def save(self, *args, **kwargs):  # noqa
                 else:
                     # There were no changes to this release
                     raise AlreadyExists("{} changed nothing - release stopped".format(self.owner))
-
         super(Release, self).save(*args, **kwargs)

rootfs/api/tests/test_config.py

@@ -279,7 +279,7 @@ def test_config_deploy_failure(self, mock_requests):
             app = App.objects.get(id=app_id)
             release = app.release_set.latest()
             self.assertEqual(release.failed, True)
-            self.assertEqual(release.exception, "error: Boom!")
+            self.assertEqual(release.conditions[0]['exception'], "Boom!")
 
     def test_invalid_config_keys(self, mock_requests):
         """Test that invalid config keys are rejected.
@@ -386,7 +386,7 @@ def test_config_failures(self, mock_requests):
             self.assertEqual(response.status_code, 201)
             self.assertEqual(app.release_set.latest().version, 4)
             self.assertEqual(app.release_set.latest().failed, True)
-            self.assertEqual(app.release_set.latest().exception, "error: Boom!")
+            self.assertEqual(app.release_set.latest().conditions[0]['exception'], "Boom!")
             self.assertEqual(app.release_set.filter(failed=False).latest().version, 3)
 
         # create a build to see that the new release is created with the last successful config
@@ -464,7 +464,7 @@ def test_unset_limits_error(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         # unset error
         body = {
             'limits': {
@@ -542,7 +542,7 @@ def test_set_config_limits_run(self, *args, **kwargs):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         body = {
             'values': json.dumps({'PORT': 5000}),
             'limits': {

rootfs/api/tests/test_gateway.py

@@ -55,7 +55,7 @@ def create_app_with_domain_and_deploy(self):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         return app_id
 
     def change_certs_auto(self, app_id, enabled):

rootfs/api/tests/test_lock.py

@@ -0,0 +1,48 @@
+import time
+from api.tests import DryccTestCase
+from api.utils import CacheLock, DeployLock
+
+
+class TestLock(DryccTestCase):
+
+    def test_cache_lock(self):
+        key = f"test_key_1_{int(time.time())}"
+        lock1 = CacheLock(key)
+        lock2 = CacheLock(key)
+        timeout = 5
+        self.assertEqual(lock1.acquire(True, timeout), True)
+        self.assertEqual(lock2.acquire(False, timeout), False)
+        time.sleep(5)
+        start_time = time.time()
+        self.assertEqual(lock2.acquire(False, timeout), True)
+        self.assertEqual(lock1.acquire(True, timeout + 1), True)
+        self.assertTrue(time.time()-start_time > 5)
+        self.assertEqual(lock2.acquire(False, timeout), False)
+        lock1.release()
+        self.assertEqual(lock2.acquire(False, timeout), True)
+
+    def test_deploy_lock(self):
+        app_id = f"test_key_1_{int(time.time())}"
+        lock1 = DeployLock(app_id)
+        lock2 = DeployLock(app_id)
+        self.assertEqual(lock1.acquire(None), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        lock1.release(None)
+        self.assertEqual(lock2.acquire(["web", "task"]), True)
+        self.assertEqual(lock1.acquire(None), False)
+        self.assertEqual(lock1.acquire(["web"]), False)
+        self.assertEqual(lock1.acquire(["bing"]), True)
+        lock2.release(["web", "task"])
+        self.assertEqual(lock1.acquire(["web"]), True)
+        self.assertEqual(lock1.acquire(["task"]), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        self.assertEqual(lock2.acquire(None), False)
+        lock2.release(["web", "task", "bing"])
+        self.assertEqual(lock2.acquire(None), True)
+        lock2.release(None)
+        self.assertEqual(lock1.acquire(["web", "task"]), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        self.assertEqual(lock2.acquire(["web", "bing"], True), True)
+        self.assertEqual(lock1.acquire(["web", "bing", "task"]), False)
+        self.assertEqual(lock1.acquire(["web", "bing", "task"], True), True)
+        lock2.release(["web", "bing", "task"])

rootfs/api/tests/test_pods.py

@@ -278,16 +278,18 @@ def test_container_errors(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 'not_an_int'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
         self.assertEqual(app.structure, {'web': 1})
 
         body = {'invalid': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
         self.assertEqual(app.structure, {'web': 1})
 
     def test_container_str(self, mock_requests):
@@ -512,7 +514,7 @@ def test_command_good(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # use `start web` for backwards compatibility with buildpacks
         self.assertEqual(release.get_deploy_args('web'), [])
@@ -571,7 +573,7 @@ def test_run_command_good(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # create a run pod
         url = f"/v2/apps/{app_id}/run"
@@ -632,7 +634,7 @@ def test_run_not_fail_on_debug(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # create a run pod
         url = f"/v2/apps/{app_id}/run"
@@ -669,7 +671,7 @@ def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # create a run pod
         url = f"/v2/apps/{app_id}/run"

rootfs/api/tests/test_release.py

@@ -105,6 +105,33 @@ def test_release(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.content)
         return release3
 
+    def test_conditions(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/builds"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker-test1': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = f'/v2/apps/{app_id}/releases/'
+        response = self.client.get(url)
+        self.assertEqual(len(response.data['results'][0]['conditions']), 1)
+
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.side_effect = Exception('Boom!')
+            url = f"/v2/apps/{app_id}/builds"
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            url = f'/v2/apps/{app_id}/releases/'
+            response = self.client.get(url)
+            self.assertEqual(response.data['results'][0]['conditions'][0]['exception'], 'Boom!')
+
     def test_get_image(self, mock_requests):
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/builds"
@@ -192,7 +219,8 @@ def test_response_data(self, mock_requests):
         response = self.client.get(url)
         for key in response.data.keys():
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'build', 'config',
-                                'summary', 'version', 'state', 'failed', 'exception'])
+                                'summary', 'version', 'state', 'failed', 'conditions',
+                                'exception'])
         expected = {
             'owner': self.user.username,
             'app': app_id,