feat(pipline): differential deployment

Author: duanhongyi

rootfs/api/management/commands/measure_loadbalancers.py

@@ -37,8 +37,9 @@ def _measure_loadbalancers(self, app_map, timestamp):
                 "usage": 1,
                 "kwargs": {
                     "ip": ip,
+                    "node": item["node"],
                     "service": item["service"],
-                    "hostname": item["hostname"],
+                    "instance": item["instance"],
                 },
                 "timestamp": start
             })

rootfs/api/models/app.py

@@ -256,15 +256,19 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
                         self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
                     if state != 'down':
                         raise DryccException(f'pipeline run state error: {state}')
-            self.log(f"{prefix} starts running pipeline.deploy")
-            self.deploy(release, None, force_deploy, rollback_on_failure)
+            procfile_types = release.diff_procfile_types()
+            if procfile_types is None or len(procfile_types) > 0:
+                self.log(f"{prefix} starts running pipeline.deploy")
+                self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
+            else:
+                self.log(f"{prefix} no changes, skip executing pipeline.deploy")
             release.state = "succeed"
         except Exception as e:
             release.state = "crashed"
             release.failed = True
             release.summary = "{} pipeline a release that failed".format(self.owner)
             release.exception = "error: {}".format(str(e))
-            self.log(f"{prefix} pipeline runtime error: {release.exception}")
+            self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
         release.save()
         self.log(f"{prefix} run completed...")
 
@@ -276,7 +280,6 @@ def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_f
         """
         if release.build is None:
             raise DryccException('No build associated with this release')
-
         # use create to make sure minimum resources are created
         self.create()
         # Previous release
@@ -330,7 +333,7 @@ def cleanup_old(self, procfile_types=None):
                 names.append(self._get_job_id(scale_type, True))
             names.append(self._get_job_id(scale_type, False))
         labels = {'heritage': 'drycc'}
-        if procfile_types:
+        if procfile_types is not None:
             labels["type__in"] = procfile_types
         deployments = self.scheduler().deployments.get(self.id, labels=labels).json()["items"]
         if deployments is not None:

rootfs/api/models/build.py

@@ -90,7 +90,8 @@ def create_release(self, user, *args, **kwargs):
                 new_release = self.app.release_set.latest()
                 new_release.state = "crashed"
                 new_release.failed = True
-                new_release.summary = "{} deployed {} which failed".format(self.owner, str(self.uuid)[:7])  # noqa
+                new_release.summary = "{} deployed {} which failed".format(
+                    self.owner, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
                 new_release.save()

rootfs/api/models/gateway.py

@@ -15,7 +15,8 @@
 DEFAULT_HTTP_PORT = 80
 DEFAULT_HTTPS_PORT = 443
 
-HOSTNAME_PROTOCOLS = ("TLS", "HTTP", "HTTPS")
+TLS_PROTOCOLS = ("HTTPS", "TLS")
+HOSTNAME_PROTOCOLS = TLS_PROTOCOLS + ("HTTP", )
 
 
 class Gateway(AuditedModel):
@@ -52,23 +53,24 @@ def listeners(self):
                 for domain in domains:
                     listener = {
                         "allowedRoutes": {"namespaces": {"from": "All"}},
-                        "name": self._get_listener_name(port, protocol, domains.index(domain)),
+                        "name": self._get_listener_name(port, protocol, domains.index(domain) + 1),
                         "port": port,
                         "hostname": domain.domain,
                         "protocol": protocol,
                     }
                     secret_name = f"{self.app.id}-auto-tls" if auto_tls else (
                         domain.certificate.name if domain.certificate else None)
-                    if secret_name:
+                    if secret_name and protocol in TLS_PROTOCOLS:
                         listener["tls"] = {
                             "certificateRefs": [{"kind": "Secret", "name": secret_name}]}
                     listeners.append(listener)
-            listeners.append({
-                "allowedRoutes": {"namespaces": {"from": "All"}},
-                "name": self._get_listener_name(port, protocol, 0),
-                "port": port,
-                "protocol": protocol,
-            })
+            if protocol not in TLS_PROTOCOLS:
+                listeners.append({
+                    "allowedRoutes": {"namespaces": {"from": "All"}},
+                    "name": self._get_listener_name(port, protocol, 0),
+                    "port": port,
+                    "protocol": protocol,
+                })
         return listeners
 
     @property

rootfs/api/models/release.py

@@ -3,6 +3,7 @@
 from django.conf import settings
 from django.db import models
 from django.contrib.auth import get_user_model
+from api.utils import dict_diff
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
@@ -108,6 +109,30 @@ def get_deploy_command(self, container_type):
         return self.build.dryccfile.get(
             'deploy', {}).get(container_type, {}).get('command', [])
 
+    def diff_procfile_types(self):
+        """
+        If returning None, it indicates that all procfile_types have changed.
+        """
+        def _get_full_deploy(release):
+            deploy = {}
+            for procfile_type, value in release.build.dryccfile['deploy'].items():
+                value['image'] = release.get_deploy_image(procfile_type)
+                value['args'] = release.get_deploy_args(procfile_type)
+                value['command'] = release.get_deploy_command(procfile_type)
+                deploy[procfile_type] = value
+            return deploy
+
+        pre_release = self.previous()
+        if (pre_release and pre_release.build and
+                pre_release.build.dryccfile and self.build and self.build.dryccfile):
+            deploy = _get_full_deploy(self)
+            pre_deploy = _get_full_deploy(pre_release)
+            procfile_types = set()
+            for value in dict_diff(deploy, pre_deploy).values():
+                procfile_types = procfile_types.union(value.keys())
+            return procfile_types
+        return None
+
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -254,7 +279,7 @@ def cleanup_old(self, procfile_types=None):
         )
         # base labels
         labels = {'heritage': 'drycc'}
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         # Cleanup controllers
         replica_sets_removal = []
@@ -286,7 +311,7 @@ def cleanup_old(self, procfile_types=None):
 
     def _cleanup_stray_pods(self, namespace, procfile_types, latest_version):
         labels = {'heritage': 'drycc'}
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         pods = self.scheduler().pod.get(namespace, labels=labels).json()['items']
         if not pods:
@@ -340,7 +365,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
             # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
             'version__notin': versions
         }
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         self.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
         secrets = self.scheduler().secret.get(namespace, labels=labels).json()['items']
@@ -361,7 +386,7 @@ def _delete_release_in_scheduler(self, namespace, procfile_types, version):
             'app': namespace,
             'version': version
         }
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         # see if the app config has deploy timeout preference, otherwise use global
         timeout = self.config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)

rootfs/api/models/service.py

@@ -59,7 +59,7 @@ def as_dict(self):
         }
 
     def port_name(self, port, protocol):
-        return "-".join([self.app.id, self.procfile_type, protocol, str(port)]).lower()
+        return "-".join([protocol, str(port)]).lower()
 
     def get_port(self, port, protocol):
         for item in self.ports: