From f5647800e9f8246885bf32813b32b0f098e1e1c7 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 6 May 2024 17:47:27 +0800
Subject: [PATCH 001/200] feat(pipline): differential deployment

---
 .../commands/measure_loadbalancers.py         |   3 +-
 rootfs/api/models/app.py                      |  13 +-
 rootfs/api/models/build.py                    |   3 +-
 rootfs/api/models/gateway.py                  |  20 +-
 rootfs/api/models/release.py                  |  33 +-
 rootfs/api/models/service.py                  |   2 +-
 rootfs/api/tests/test_gateway.py              | 200 ++++--------
 rootfs/api/tests/test_release.py              | 287 ++++++++++++++++++
 rootfs/api/views.py                           |   5 +-
 9 files changed, 394 insertions(+), 172 deletions(-)

diff --git a/rootfs/api/management/commands/measure_loadbalancers.py b/rootfs/api/management/commands/measure_loadbalancers.py
index 68f5fa83d..e9c295028 100644
--- a/rootfs/api/management/commands/measure_loadbalancers.py
+++ b/rootfs/api/management/commands/measure_loadbalancers.py
@@ -37,8 +37,9 @@ def _measure_loadbalancers(self, app_map, timestamp):
                 "usage": 1,
                 "kwargs": {
                     "ip": ip,
+                    "node": item["node"],
                     "service": item["service"],
-                    "hostname": item["hostname"],
+                    "instance": item["instance"],
                 },
                 "timestamp": start
             })
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index dcbe069a1..50d356067 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -256,15 +256,19 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
                         self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
                     if state != 'down':
                         raise DryccException(f'pipeline run state error: {state}')
-            self.log(f"{prefix} starts running pipeline.deploy")
-            self.deploy(release, None, force_deploy, rollback_on_failure)
+            procfile_types = release.diff_procfile_types()
+            if procfile_types is None or len(procfile_types) > 0:
+                self.log(f"{prefix} starts running pipeline.deploy")
+                self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
+            else:
+                self.log(f"{prefix} no changes, skip executing pipeline.deploy")
             release.state = "succeed"
         except Exception as e:
             release.state = "crashed"
             release.failed = True
             release.summary = "{} pipeline a release that failed".format(self.owner)
             release.exception = "error: {}".format(str(e))
-            self.log(f"{prefix} pipeline runtime error: {release.exception}")
+            self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
         release.save()
         self.log(f"{prefix} run completed...")
 
@@ -276,7 +280,6 @@ def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_f
         """
         if release.build is None:
             raise DryccException('No build associated with this release')
-
         # use create to make sure minimum resources are created
         self.create()
         # Previous release
@@ -330,7 +333,7 @@ def cleanup_old(self, procfile_types=None):
                 names.append(self._get_job_id(scale_type, True))
             names.append(self._get_job_id(scale_type, False))
         labels = {'heritage': 'drycc'}
-        if procfile_types:
+        if procfile_types is not None:
             labels["type__in"] = procfile_types
         deployments = self.scheduler().deployments.get(self.id, labels=labels).json()["items"]
         if deployments is not None:
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 0af54e2dc..3f7d0ba21 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -90,7 +90,8 @@ def create_release(self, user, *args, **kwargs):
                 new_release = self.app.release_set.latest()
                 new_release.state = "crashed"
                 new_release.failed = True
-                new_release.summary = "{} deployed {} which failed".format(self.owner, str(self.uuid)[:7])  # noqa
+                new_release.summary = "{} deployed {} which failed".format(
+                    self.owner, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
                 new_release.save()
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 4703210b3..355495dd0 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -15,7 +15,8 @@
 DEFAULT_HTTP_PORT = 80
 DEFAULT_HTTPS_PORT = 443
 
-HOSTNAME_PROTOCOLS = ("TLS", "HTTP", "HTTPS")
+TLS_PROTOCOLS = ("HTTPS", "TLS")
+HOSTNAME_PROTOCOLS = TLS_PROTOCOLS + ("HTTP", )
 
 
 class Gateway(AuditedModel):
@@ -52,23 +53,24 @@ def listeners(self):
                 for domain in domains:
                     listener = {
                         "allowedRoutes": {"namespaces": {"from": "All"}},
-                        "name": self._get_listener_name(port, protocol, domains.index(domain)),
+                        "name": self._get_listener_name(port, protocol, domains.index(domain) + 1),
                         "port": port,
                         "hostname": domain.domain,
                         "protocol": protocol,
                     }
                     secret_name = f"{self.app.id}-auto-tls" if auto_tls else (
                         domain.certificate.name if domain.certificate else None)
-                    if secret_name:
+                    if secret_name and protocol in TLS_PROTOCOLS:
                         listener["tls"] = {
                             "certificateRefs": [{"kind": "Secret", "name": secret_name}]}
                     listeners.append(listener)
-            listeners.append({
-                "allowedRoutes": {"namespaces": {"from": "All"}},
-                "name": self._get_listener_name(port, protocol, 0),
-                "port": port,
-                "protocol": protocol,
-            })
+            if protocol not in TLS_PROTOCOLS:
+                listeners.append({
+                    "allowedRoutes": {"namespaces": {"from": "All"}},
+                    "name": self._get_listener_name(port, protocol, 0),
+                    "port": port,
+                    "protocol": protocol,
+                })
         return listeners
 
     @property
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 03b30c73c..978df1fa2 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -3,6 +3,7 @@
 from django.conf import settings
 from django.db import models
 from django.contrib.auth import get_user_model
+from api.utils import dict_diff
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
@@ -108,6 +109,30 @@ def get_deploy_command(self, container_type):
         return self.build.dryccfile.get(
             'deploy', {}).get(container_type, {}).get('command', [])
 
+    def diff_procfile_types(self):
+        """
+        If returning None, it indicates that all procfile_types have changed.
+        """
+        def _get_full_deploy(release):
+            deploy = {}
+            for procfile_type, value in release.build.dryccfile['deploy'].items():
+                value['image'] = release.get_deploy_image(procfile_type)
+                value['args'] = release.get_deploy_args(procfile_type)
+                value['command'] = release.get_deploy_command(procfile_type)
+                deploy[procfile_type] = value
+            return deploy
+
+        pre_release = self.previous()
+        if (pre_release and pre_release.build and
+                pre_release.build.dryccfile and self.build and self.build.dryccfile):
+            deploy = _get_full_deploy(self)
+            pre_deploy = _get_full_deploy(pre_release)
+            procfile_types = set()
+            for value in dict_diff(deploy, pre_deploy).values():
+                procfile_types = procfile_types.union(value.keys())
+            return procfile_types
+        return None
+
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -254,7 +279,7 @@ def cleanup_old(self, procfile_types=None):
         )
         # base labels
         labels = {'heritage': 'drycc'}
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         # Cleanup controllers
         replica_sets_removal = []
@@ -286,7 +311,7 @@ def cleanup_old(self, procfile_types=None):
 
     def _cleanup_stray_pods(self, namespace, procfile_types, latest_version):
         labels = {'heritage': 'drycc'}
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         pods = self.scheduler().pod.get(namespace, labels=labels).json()['items']
         if not pods:
@@ -340,7 +365,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
             # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
             'version__notin': versions
         }
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         self.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
         secrets = self.scheduler().secret.get(namespace, labels=labels).json()['items']
@@ -361,7 +386,7 @@ def _delete_release_in_scheduler(self, namespace, procfile_types, version):
             'app': namespace,
             'version': version
         }
-        if procfile_types:
+        if procfile_types is not None:
             labels['type__in'] = procfile_types
         # see if the app config has deploy timeout preference, otherwise use global
         timeout = self.config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index 21218dc93..c0a00f70e 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -59,7 +59,7 @@ def as_dict(self):
         }
 
     def port_name(self, port, protocol):
-        return "-".join([self.app.id, self.procfile_type, protocol, str(port)]).lower()
+        return "-".join([protocol, str(port)]).lower()
 
     def get_port(self, port, protocol):
         for item in self.ports:
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 2b8cc0794..4ea4edf7d 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -174,39 +174,14 @@ def test_add_tls_listener(self):
         name = "tls-gateway"
         _, domain, secret_name, results = self.add_tls_listener(name, "TLS", port)
         expect = [{
-            'app': name,
-            'owner': 'autotest',
-            'name': name,
+            'app': name, 'owner': 'autotest', 'name': name,
             'listeners': [{
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'tcp-%s-0' % port,
-                'port': port,
-                'hostname': domain,
+                'allowedRoutes': {'namespaces': {'from': 'All'}},
+                'name': 'tcp-443-1', 'port': 443, 'hostname': domain,
                 'protocol': 'TLS',
-                'tls': {
-                    'certificateRefs': [{
-                        'kind': 'Secret',
-                        'name': secret_name
-                    }]
-                }
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'tcp-%s-0' % port,
-                'port': port,
-                'protocol': 'TLS'
+                'tls': {'certificateRefs': [{'kind': 'Secret', 'name': secret_name}]}
             }],
-            'addresses': [{
-                'type': 'IPAddress',
-                'value': '172.22.108.207'
-            }]
+            'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
         }]
         self.assertEqual(results, expect)
 
@@ -224,7 +199,7 @@ def test_add_https_listener(self):
                         'from': 'All'
                     }
                 },
-                'name': 'mix-%s-0' % port,
+                'name': 'mix-%s-1' % port,
                 'port': port,
                 'hostname': domain,
                 'protocol': 'HTTPS',
@@ -234,15 +209,6 @@ def test_add_https_listener(self):
                         'name': secret_name
                     }]
                 }
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'mix-%s-0' % port,
-                'port': port,
-                'protocol': 'HTTPS'
             }],
             'addresses': [{
                 'type': 'IPAddress',
@@ -256,64 +222,31 @@ def test_add_http_listener(self):
         name = "bingo-gateway"
         _, domain, secret_name, results = self.add_tls_listener(name, "HTTP", port)
         expect = [{
-            'app': 'bingo-gateway',
-            'owner': 'autotest',
-            'name': 'bingo-gateway',
-            'listeners': [{
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'tcp-%s-0' % port,
-                'port': port,
-                'hostname': domain,
-                'protocol': 'HTTP',
-                'tls': {
-                    'certificateRefs': [{
-                        'kind': 'Secret',
-                        'name': secret_name
-                    }]
-                }
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
+            'app': 'bingo-gateway', 'owner': 'autotest', 'name': 'bingo-gateway',
+            'listeners': [
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-1',
+                    'port': 80,
+                    'hostname': domain,
+                    'protocol': 'HTTP'
                 },
-                'name': 'tcp-%s-0' % port,
-                'port': port,
-                'protocol': 'HTTP'
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-0',
+                    'port': 80,
+                    'protocol': 'HTTP'
                 },
-                'name': 'mix-443-0',
-                'port': 443,
-                'hostname': 'autotest.example.com',
-                'protocol': 'HTTPS',
-                'tls': {
-                    'certificateRefs': [{
-                        'kind': 'Secret',
-                        'name': secret_name
-                    }]
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'mix-443-1',
+                    'port': 443,
+                    'hostname': domain,
+                    'protocol': 'HTTPS',
+                    'tls': {'certificateRefs': [{'kind': 'Secret', 'name': secret_name}]}
                 }
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'mix-443-0',
-                'port': 443,
-                'protocol': 'HTTPS'
-            }],
-            'addresses': [{
-                'type': 'IPAddress',
-                'value': '172.22.108.207'
-            }]
+            ],
+            'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
         }]
         self.assertEqual(results, expect)
 
@@ -356,7 +289,7 @@ def test_remove_domain(self):
     def test_remove_tls(self):
         app_id, domain, secret_name, _ = self.add_tls_listener("bingo-gateway", "HTTPS", 443)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
-        self.assertEqual(len(response.data["results"][0]["listeners"]), 2)
+        self.assertEqual(len(response.data["results"][0]["listeners"]), 1)
         response = self.client.delete(
             '{}/{}/domain/{}/'.format('/v2/certs', secret_name, domain)
         )
@@ -373,7 +306,7 @@ def test_certs_auto_enabled(self):
             data)
         self.assertEqual(response.status_code, 201, response.data)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
-        self.assertEqual(len(response.data["results"][0]["listeners"]), 2)
+        self.assertEqual(len(response.data["results"][0]["listeners"]), 1)
 
     def test_remove_listener(self):
         app_id = self.create_app()
@@ -412,61 +345,30 @@ def test_change_tls(self):
             'app': app_id,
             'owner': 'autotest',
             'name': app_id,
-            'listeners': [{
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'tcp-80-0',
-                'port': 80,
-                'hostname': 'test-domain.example.com',
-                'protocol': 'HTTP',
-                'tls': {
-                    'certificateRefs': [{
-                        'kind': 'Secret',
-                        'name': '%s-auto-tls' % app_id
-                    }]
-                }
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
+            'listeners': [
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-1',
+                    'port': 80,
+                    'hostname': 'test-domain.example.com',
+                    'protocol': 'HTTP'
                 },
-                'name': 'tcp-80-0',
-                'port': 80,
-                'protocol': 'HTTP'
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-0',
+                    'port': 80,
+                    'protocol': 'HTTP'
                 },
-                'name': 'mix-443-0',
-                'port': 443,
-                'hostname': 'test-domain.example.com',
-                'protocol': 'HTTPS',
-                'tls': {
-                    'certificateRefs': [{
-                        'kind': 'Secret',
-                        'name': '%s-auto-tls' % app_id
-                    }]
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'mix-443-1',
+                    'port': 443,
+                    'hostname': 'test-domain.example.com',
+                    'protocol': 'HTTPS',
+                    'tls': {'certificateRefs': [{'kind': 'Secret', 'name': f'{app_id}-auto-tls'}]}
                 }
-            }, {
-                'allowedRoutes': {
-                    'namespaces': {
-                        'from': 'All'
-                    }
-                },
-                'name': 'mix-443-0',
-                'port': 443,
-                'protocol': 'HTTPS'
-            }],
-            'addresses': [{
-                'type': 'IPAddress',
-                'value': '172.22.108.207'
-            }]
+            ],
+            'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
         }]
         self.assertEqual(response.data["results"], expect, response.data)
 
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index f4f09b355..1dbcf572d 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -11,6 +11,7 @@
 from unittest import mock
 
 from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.build import Build
 from api.models.release import Release
 from scheduler import KubeHTTPException
 from api.exceptions import DryccException
@@ -612,3 +613,289 @@ def test_release_external_registry_no_port(self, mock_requests):
         release = app.release_set.latest()
         port = release.get_port()
         self.assertIsNone(port)
+
+    def test_diff_procfile_types(self, mock_requests):
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        user = User.objects.get(username='autotest')
+
+        # CNCF Buildpack app
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "build": {
+                    "docker": {
+                        "web": "Dockerfile",
+                        "worker": "worker/Dockerfile",
+                    }
+                },
+                "deploy": {
+                    "web": {
+                        "command": ["bash", "-ec"],
+                        "args": ["bundle exec puma -C config/puma.rb"]
+                    },
+                    "worker": {
+                        "command": ["bash", "-ec"],
+                        "args": ["python myworker.py"]
+                    }
+                }
+            },
+        )
+
+        # create an initial release
+        release = Release.objects.create(
+            version=2,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertIsNone(release.diff_procfile_types())
+        # test has image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "deploy": {
+                    "web": {
+                        "image": "docker.io/test/test:v1",
+                        "command": ["bash", "-ec"],
+                        "args": ["bundle exec puma -C config/puma.rb"]
+                    },
+                    "worker": {
+                        "image": "docker.io/test/test:v1",
+                        "command": ["bash", "-ec"],
+                        "args": ["python myworker.py"]
+                    }
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=3,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), {'web', 'worker'})
+        # test has image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "deploy": {
+                    "web": {
+                        "image": "docker.io/test/test:v2",
+                        "command": ["bash", "-ec"],
+                        "args": ["bundle exec puma -C config/puma.rb"]
+                    },
+                    "worker": {
+                        "image": "docker.io/test/test:v1",
+                        "command": ["bash", "-ec"],
+                        "args": ["python myworker.py"]
+                    }
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=4,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), {'web'})
+        # test has image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "deploy": {
+                    "web": {
+                        "image": "docker.io/test/test:v2",
+                        "command": ["bash", "-ec"],
+                        "args": ["bundle exec puma -C config/puma.rb"]
+                    },
+                    "worker": {
+                        "image": "docker.io/test/test:v2",
+                        "command": ["bash", "-ec"],
+                        "args": ["python myworker.py"]
+                    }
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=5,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), {'worker'})
+        # test no image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "build": {
+                    "docker": {
+                        "web": "Dockerfile",
+                        "worker": "worker/Dockerfile",
+                    }
+                },
+                "deploy": {
+                    "web": {
+                        "command": ["bash", "-ec"],
+                        "args": ["bundle exec puma -C config/puma.rb"]
+                    },
+                    "worker": {
+                        "command": ["bash", "-ec"],
+                        "args": ["python myworker.py"]
+                    },
+                    "worker-sync": {
+                        "image": "web",
+                        "command": ["bash", "-ec"],
+                        "args": ["python myworker.py sync"]
+                    },
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=6,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), {'web', 'worker', 'worker-sync'})
+        # test has image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "run": {
+                    "args": ["sleep", "60s"],
+                    "image": "registry.drycc.cc/drycc/base:bookworm"
+                },
+                "deploy": {
+                    "web": {
+                        "image": "registry.drycc.cc/drycc/python-dev",
+                        "args": ["python", "-m", "http.server", "5000"]
+                    },
+                    "task": {
+                        "image": "docker.io/library/nginx",
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    }
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=7,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), {'worker', 'task', 'worker-sync', 'web'})
+        # test has image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "run": {
+                    "args": ["sleep", "60s"],
+                    "image": "registry.drycc.cc/drycc/base:bookworm"
+                },
+                "deploy": {
+                    "web": {
+                        "image": "registry.drycc.cc/drycc/python-dev",
+                        "args": ["python", "-m", "http.server", "5000"]
+                    },
+                    "task": {
+                        "image": "docker.io/library/nginx:mainline-bookworm-perl",
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    }
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=8,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), {'task'})
+        # test has image
+        build = Build.objects.create(
+            owner=user,
+            app=app,
+            image="qwerty",
+            procfile={},
+            sha='african-swallow',
+            dockerfile={},
+            dryccfile={
+                "run": {
+                    "args": ["sleep", "60s"],
+                    "image": "registry.drycc.cc/drycc/base:bookworm"
+                },
+                "deploy": {
+                    "web": {
+                        "image": "registry.drycc.cc/drycc/python-dev",
+                        "args": ["python", "-m", "http.server", "5000"]
+                    },
+                    "task": {
+                        "image": "docker.io/library/nginx:mainline-bookworm-perl",
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    }
+                }
+            },
+        )
+        release = Release.objects.create(
+            version=9,
+            owner=user,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        self.assertEqual(release.diff_procfile_types(), set())
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            with mock.patch('api.models.app.logger') as mock_logger:
+                mock_kube.return_value = ['up', 'down']
+                app.pipeline(release, False, True)
+                self.assertEqual(release.state, "succeed")
+                prefix = f"[{release.app.id}]: [pipeline] release v{release.version}"
+                exp_msg = f"{prefix} no changes, skip executing pipeline.deploy"
+                mock_logger.log.assert_any_call(logging.INFO, exp_msg)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 7539f7ce0..5fde0dc0e 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -419,8 +419,9 @@ def post_save(self, config):
                     if field in config.procfile_fields:
                         for value in diff.values():
                             procfile_types.update(value.keys())
-                config.app.deploy(
-                    release, procfile_types=list(procfile_types) if procfile_types else None)
+                # all_diff_fields changed, deploy all.
+                procfile_types = procfile_types if procfile_types else None
+                config.app.deploy(release, procfile_types)
             release.state = "succeed"
             release.save()
         except Exception as e:

From efdf8e362365735803a54cdaaad09964763d0a45 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 9 May 2024 18:21:20 +0800
Subject: [PATCH 002/200] fix(logger): miss log

---
 rootfs/api/models/gateway.py | 30 ++++++++++++++++++++++++++----
 rootfs/api/models/tls.py     |  4 ++--
 rootfs/requirements.txt      |  2 --
 3 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 355495dd0..cb5b643c2 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -25,6 +25,17 @@ class Gateway(AuditedModel):
     name = models.CharField(max_length=63, db_index=True)
     ports = models.JSONField(default=list)
 
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this service.
+
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
+
     def add(self, port, protocol):
         # check port
         if not self._check_port(port, protocol):
@@ -126,8 +137,8 @@ def delete(self, *args, **kwargs):
         try:
             self.scheduler().gateways.delete(self.app.id, self.name, ignore_exception=False)
         except KubeException:
-            logger.log(
-                msg='Kubernetes gateway cannot be deleted: {}'.format(self.name),
+            self.log(
+                'Kubernetes gateway cannot be deleted: {}'.format(self.name),
                 level=logging.ERROR,
             )
         return super().delete(*args, **kwargs)
@@ -225,6 +236,17 @@ def current_rules(self):
             return self.rules["canary"]
         return self.rules["stable"]
 
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this service.
+
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
+
     def check_rules(self):
         service = self.app.service_set.filter(
             procfile_type=self.procfile_type).first()
@@ -297,8 +319,8 @@ def delete(self, *args, **kwargs):
             k8s_route = getattr(self.scheduler(), self.kind.lower())
             k8s_route.delete(self.app.id, self.name, ignore_exception=False)
         except KubeException:
-            logger.log(
-                msg='Kubernetes {} cannot be deleted: {}'.format(self.kind.lower(), self.name),
+            self.log(
+                'Kubernetes {} cannot be deleted: {}'.format(self.kind.lower(), self.name),
                 level=logging.ERROR,
             )
         return super().delete(*args, **kwargs)
diff --git a/rootfs/api/models/tls.py b/rootfs/api/models/tls.py
index 68fbb4332..828269f69 100644
--- a/rootfs/api/models/tls.py
+++ b/rootfs/api/models/tls.py
@@ -106,8 +106,8 @@ def refresh_certificate_to_k8s(self):
                     version = data["metadata"]["resourceVersion"]
                     self.scheduler().certificate.put(namespace, name, hosts, version)
                 else:
-                    logger.log(
-                        msg="certificate {} does not exist".format(namespace), level=logging.INFO)
+                    self.log(
+                        "certificate {} does not exist".format(namespace), level=logging.INFO)
                     self.scheduler().certificate.create(namespace, name, hosts)
             else:
                 self.log("skip creating certificate, no domain name set", logging.WARNING)
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 2d0c52e64..b2a509996 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -27,6 +27,4 @@ hiredis==2.3.2
 django_redis==5.4.0
 dj-database-url==2.1.0
 social-auth-app-django==5.4.1
-python-jose==3.3.0
-Authlib==1.3.0
 kubernetes==29.0.0

From f93fc67194dcda89ccba6bbd6ce79e77d1ab6cda Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 9 May 2024 21:41:44 +0800
Subject: [PATCH 003/200] fix(release): secret env miss

---
 rootfs/api/models/release.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 978df1fa2..e5fd134c2 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -343,7 +343,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
         """
 
         # Find all ReplicaSets
-        versions = []
+        versions = ['v{}'.format(self.version), ]
         labels = {'heritage': 'drycc', 'app': namespace}
         replicasets = self.scheduler().rs.get(namespace, labels=labels).json()['items']
         if not replicasets:

From a48ae7dea3126f82da482802cd43c229d17338c7 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 10 May 2024 00:14:44 +0800
Subject: [PATCH 004/200] chore(release): add version_name

---
 rootfs/api/models/app.py     | 13 +++++------
 rootfs/api/models/release.py | 44 ++++++++++++++++++++----------------
 rootfs/api/signals.py        |  2 +-
 3 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 50d356067..013f95fe0 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -240,7 +240,7 @@ def scale(self, user, structure):
         self._scale(user, structure, release, app_settings)
 
     def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
-        prefix = f"[pipeline] release v{release.version}"
+        prefix = f"[pipeline] release {release.version_name}"
         try:
             self.log(f"{prefix} starts running...")
             if release.build.dryccfile:
@@ -518,9 +518,8 @@ def set_application_config(self, release, procfile_type):
         updates it if it already exists
         """
         # env vars are stored in secrets and mapped to env in k8s
-        version = 'v{}'.format(release.version)
         labels = {
-            'version': version,
+            'version': release.version_name,
             'type': procfile_type,
             'class': 'env'
         }
@@ -533,7 +532,7 @@ def set_application_config(self, release, procfile_type):
         # dictionary sorted by key
         secrets_env = OrderedDict(sorted(secrets_env.items(), key=lambda t: t[0]))
 
-        secret_name = "{}-{}-{}-env".format(self.id, procfile_type, version)
+        secret_name = "{}-{}-{}-env".format(self.id, procfile_type, release.version_name)
         try:
             self.scheduler().secret.get(self.id, secret_name)
         except KubeHTTPException:
@@ -645,7 +644,7 @@ def _deploy(self, deploys, procfile_types, prev_release,
                 if (prev_release.canary == release.canary and
                         rollback_on_failure and prev_release.build is not None):
                     err = 'There was a problem deploying {}. Rolling back to release {}.'.format(
-                        'v{}'.format(release.version), "v{}".format(prev_release.version))
+                        release.version_name, prev_release.version_name)
                     # This goes in the log before the rollback starts
                     self.log(err, logging.ERROR)
                     # revert all process types to old release
@@ -925,7 +924,7 @@ def _build_env_vars(self, release, procfile_type):
         # mix in default environment information drycc may require
         default_env = {
             'DRYCC_APP': self.id,
-            'WORKFLOW_RELEASE': 'v{}'.format(release.version),
+            'WORKFLOW_RELEASE': release.version_name,
             'WORKFLOW_RELEASE_SUMMARY': release.summary,
             'WORKFLOW_RELEASE_CREATED_AT': str(release.created.strftime(
                 settings.DRYCC_DATETIME_FORMAT))
@@ -1042,7 +1041,7 @@ def _gather_app_settings(self, release, app_settings, procfile_type, replicas, v
             'envs': envs,
             'registry': config.registry,
             'replicas': replicas,
-            'version': 'v{}'.format(release.version),
+            'version': release.version_name,
             'app_type': procfile_type,
             'resources': {"limits": limit_plan.limits, "requests": limit_plan.requests},
             'build_type': release.build.type,
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index e5fd134c2..879928012 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -44,7 +44,7 @@ class Meta:
         unique_together = (('app', 'version'),)
 
     def __str__(self):
-        return "{0}-v{1}".format(self.app.id, self.version)
+        return "{0}-{1}".format(self.app.id, self.version_name)
 
     @property
     def procfile_types(self):
@@ -52,6 +52,10 @@ def procfile_types(self):
             return self.build.procfile_types
         return []
 
+    @property
+    def version_name(self):
+        return f'v{self.version}'
+
     def get_run_image(self):
         """
         In the run phase of dryccfile
@@ -272,9 +276,8 @@ def cleanup_old(self, procfile_types=None):
         Secrets no longer tied to any ReplicaSet
         Stray pods no longer relevant to the latest release
         """
-        latest_version = 'v{}'.format(self.version)
         self.log(
-            'Cleaning up RSs for releases older than {} (latest)'.format(latest_version),
+            'Cleaning up RSs for releases older than {} (latest)'.format(self.version_name),
             level=logging.DEBUG
         )
         # base labels
@@ -287,29 +290,30 @@ def cleanup_old(self, procfile_types=None):
         if not replica_sets:
             replica_sets = []
         for replica_set in replica_sets:
-            current_version = replica_set['metadata']['labels']['version']
+            current_version_name = replica_set['metadata']['labels']['version']
             # skip the latest release
-            if current_version == latest_version:
+            if current_version_name == self.version_name:
                 continue
 
             # aggregate versions together to removal all at once
-            if current_version not in replica_sets_removal:
-                replica_sets_removal.append(current_version)
+            if current_version_name not in replica_sets_removal:
+                replica_sets_removal.append(current_version_name)
 
         if replica_sets_removal:
             self.log(
-                'Found the following versions to cleanup: {}'.format(', '.join(replica_sets_removal)),  # noqa
+                'Found the following versions to cleanup: {}'.format(
+                    ', '.join(replica_sets_removal)),
                 level=logging.DEBUG
             )
         # this is RC related
-        for version in replica_sets_removal:
-            self._delete_release_in_scheduler(self.app.id, procfile_types, version)
+        for version_name in replica_sets_removal:
+            self._delete_release_in_scheduler(self.app.id, procfile_types, version_name)
         # handle Deployments specific cleanups
         self._cleanup_deployment_secrets_and_configs(self.app.id, procfile_types)
         # Remove stray pods
-        self._cleanup_stray_pods(self.app.id, procfile_types, latest_version)
+        self._cleanup_stray_pods(self.app.id, procfile_types, self.version_name)
 
-    def _cleanup_stray_pods(self, namespace, procfile_types, latest_version):
+    def _cleanup_stray_pods(self, namespace, procfile_types, latest_version_name):
         labels = {'heritage': 'drycc'}
         if procfile_types is not None:
             labels['type__in'] = procfile_types
@@ -320,9 +324,9 @@ def _cleanup_stray_pods(self, namespace, procfile_types, latest_version):
             if self.scheduler().pod.deleted(pod):
                 continue
 
-            current_version = pod['metadata']['labels']['version']
+            current_version_name = pod['metadata']['labels']['version']
             # skip the latest release
-            if current_version == latest_version:
+            if current_version_name == latest_version_name:
                 continue
 
             try:
@@ -343,7 +347,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
         """
 
         # Find all ReplicaSets
-        versions = ['v{}'.format(self.version), ]
+        version_names = [self.version_name, ]
         labels = {'heritage': 'drycc', 'app': namespace}
         replicasets = self.scheduler().rs.get(namespace, labels=labels).json()['items']
         if not replicasets:
@@ -351,11 +355,11 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
         for replicaset in replicasets:
             if (
                 'version' not in replicaset['metadata']['labels'] or
-                replicaset['metadata']['labels']['version'] in versions
+                replicaset['metadata']['labels']['version'] in version_names
             ):
                 continue
 
-            versions.append(replicaset['metadata']['labels']['version'])
+            version_names.append(replicaset['metadata']['labels']['version'])
 
         # find all env secrets not owned by any replicaset
         labels = {
@@ -363,7 +367,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
             'app': namespace,
             'class': 'env',
             # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
-            'version__notin': versions
+            'version__notin': version_names
         }
         if procfile_types is not None:
             labels['type__in'] = procfile_types
@@ -374,7 +378,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
         for secret in secrets:
             self.scheduler().secret.delete(namespace, secret['metadata']['name'])
 
-    def _delete_release_in_scheduler(self, namespace, procfile_types, version):
+    def _delete_release_in_scheduler(self, namespace, procfile_types, version_name):
         """
         Deletes a specific release in k8s based on ReplicationController
 
@@ -384,7 +388,7 @@ def _delete_release_in_scheduler(self, namespace, procfile_types, version):
         labels = {
             'heritage': 'drycc',
             'app': namespace,
-            'version': version
+            'version': version_name
         }
         if procfile_types is not None:
             labels['type__in'] = procfile_types
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index 6ac1a9be9..0301a8654 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -90,7 +90,7 @@ def _hook_release_created(**kwargs):
             url = deploy_hook
             params = {
                 'app': release.app,
-                'release': 'v{}'.format(release.version),
+                'release': release.version_name,
                 'release_summary': release.summary,
                 'sha': '',
                 'user': release.owner,

From 8aeab3cb65f8961db118e319f9565c3b8598ca6a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 10 May 2024 15:37:10 +0800
Subject: [PATCH 005/200] feat(k8s): differential update state to k8s

---
 .../commands/load_db_state_to_k8s.py          | 17 ++----
 rootfs/api/models/app.py                      | 27 ++++++++++
 rootfs/scheduler/resources/deployment.py      | 53 ++++++++++---------
 3 files changed, 59 insertions(+), 38 deletions(-)

diff --git a/rootfs/api/management/commands/load_db_state_to_k8s.py b/rootfs/api/management/commands/load_db_state_to_k8s.py
index cb0cf384f..ebce06e61 100644
--- a/rootfs/api/management/commands/load_db_state_to_k8s.py
+++ b/rootfs/api/management/commands/load_db_state_to_k8s.py
@@ -20,14 +20,9 @@ class Command(BaseCommand):
     to k8s.
     """
 
-    def _deploy(self, app, release):
-        if release.build is None:
-            print('WARNING: {} has no build associated with '
-                  'its latest release. Skipping deployment...'.format(app))
-            return
-
+    def _deploy(self, app):
         try:
-            app.deploy(release)
+            app.state_to_k8s()
         except AlreadyExists as error:
             logger.debug(error)
             print('WARNING: {} has a deployment in progress. '
@@ -50,13 +45,9 @@ def handle(self, *args, **options):
                 cert.attach_in_kubernetes(domain)
 
         # deploy apps
-        print("Deploying available applications")
+        print("Deploying available applications.")
         for app in App.objects.all():
-            release = app.release_set.filter(failed=False).latest()
-            if release.canary:
-                self._deploy(app, app.release_set.filter(failed=False, canary=False).latest())
-            self._deploy(app, release)
-
+            self._deploy(app)
         print("Done Publishing DB state to kubernetes.")
 
     def save_apps(self):
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 013f95fe0..fa7253818 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -512,6 +512,33 @@ def image_pull_secret(self, namespace, registry, image):
 
         return name
 
+    def state_to_k8s(self):
+        def _load_procfile_types(canary):
+            procfile_types = set()
+            for procfile_type, scale in self.structure.items():
+                response = self.scheduler().deployment.get(
+                    self.id, self._get_job_id(procfile_type, canary),
+                    ignore_exception=True)
+                if response.status_code == 404 and scale > 0:
+                    procfile_types.add(procfile_type)
+                elif response.status_code != 200:
+                    data = response.json()
+                    self.log('get deployment status_code {}, message: {}'.format(
+                        response.status_code, data.get("message", "")), logging.ERROR)
+            return procfile_types
+
+        release = self.release_set.filter(failed=False).latest()
+        if release.build is None:
+            self.log('the last release does not have a build, skipping deployment...')
+            return
+        procfile_types = _load_procfile_types(False)
+        if release.canary:
+            procfile_types = procfile_types.union(_load_procfile_types(canary=True))
+        if len(procfile_types) == 0:
+            self.log('the cluster status is the latest, skipping deployment...')
+            return
+        self.deploy(release, procfile_types)
+
     def set_application_config(self, release, procfile_type):
         """
         Creates the application config as a secret in Kubernetes and
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index 43f269377..21745ba40 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -9,7 +9,7 @@ class Deployment(Resource):
     api_prefix = 'apis'
     api_version = 'apps/v1'
 
-    def get(self, namespace, name=None, **kwargs):
+    def get(self, namespace, name=None, ignore_exception=False, **kwargs):
         """
         Fetch a single Deployment or a list
         """
@@ -24,7 +24,7 @@ def get(self, namespace, name=None, **kwargs):
 
         url = self.api(url, *args)
         response = self.http_get(url, params=self.query_params(**kwargs))
-        if self.unhealthy(response.status_code):
+        if self.unhealthy(response.status_code) and not ignore_exception:
             args.reverse()  # error msg is in reverse order
             raise KubeHTTPException(response, message, *args)
 
@@ -115,7 +115,8 @@ def manifest(self, namespace, name, image, command, args, spec_annotations, **kw
 
         return manifest
 
-    def create(self, namespace, name, image, command, args, spec_annotations, **kwargs):
+    def create(self, namespace, name, image, command, args,
+               spec_annotations, ignore_exception=False, **kwargs):
         manifest = self.manifest(namespace, name, image,
                                  command, args, spec_annotations, **kwargs)
 
@@ -123,17 +124,18 @@ def create(self, namespace, name, image, command, args, spec_annotations, **kwar
         response = self.http_post(url, json=manifest)
         if self.unhealthy(response.status_code):
             self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
-            raise KubeHTTPException(
-                response,
-                'create Deployment "{}" in Namespace "{}"', name, namespace
-            )
-
-        self.wait_until_updated(namespace, name)
-        self.wait_until_ready(namespace, name, **kwargs)
-
+            if not ignore_exception:
+                raise KubeHTTPException(
+                    response,
+                    'create Deployment "{}" in Namespace "{}"', name, namespace
+                )
+        else:
+            self.wait_until_updated(namespace, name)
+            self.wait_until_ready(namespace, name, **kwargs)
         return response
 
-    def update(self, namespace, name, image, command, args, spec_annotations, **kwargs):
+    def update(self, namespace, name, image, command, args,
+               spec_annotations, ignore_exception=False, **kwargs):
         manifest = self.manifest(namespace, name, image,
                                  command, args, spec_annotations, **kwargs)
 
@@ -141,14 +143,15 @@ def update(self, namespace, name, image, command, args, spec_annotations, **kwar
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
             self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
-            raise KubeHTTPException(response, 'update Deployment "{}"', name)
-
-        self.wait_until_updated(namespace, name)
-        self.wait_until_ready(namespace, name, **kwargs)
-
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'update Deployment "{}"', name)
+        else:
+            self.wait_until_updated(namespace, name)
+            self.wait_until_ready(namespace, name, **kwargs)
         return response
 
-    def patch(self, namespace, name, image, command, args, spec_annotations, **kwargs):
+    def patch(self, namespace, name, image, command, args,
+              spec_annotations, ignore_exception=False, **kwargs):
         manifest = self.manifest(namespace, name, image,
                                  command, args, spec_annotations, **kwargs)
 
@@ -161,11 +164,11 @@ def patch(self, namespace, name, image, command, args, spec_annotations, **kwarg
 
         if self.unhealthy(response.status_code):
             self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
-            raise KubeHTTPException(response, 'patch Deployment "{}"', name)
-
-        self.wait_until_updated(namespace, name)
-        self.wait_until_ready(namespace, name, **kwargs)
-
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'patch Deployment "{}"', name)
+        else:
+            self.wait_until_updated(namespace, name)
+            self.wait_until_ready(namespace, name, **kwargs)
         return response
 
     def delete(self, namespace, name, ignore_exception=False):
@@ -200,7 +203,7 @@ def scale(self, namespace, name, **kwargs):
             kwargs['previous_replicas'] = current
             self.wait_until_ready(namespace, name, **kwargs)
 
-    def restart(self, namespace, name):
+    def restart(self, namespace, name, ignore_exception=False):
         url = self.api(
             "/namespaces/{}/deployments/{}?fieldManager=kubectl-rollout&pretty=true",
             namespace, name
@@ -221,7 +224,7 @@ def restart(self, namespace, name):
             }),
             headers={"Content-Type": "application/merge-patch+json"},
         )
-        if self.unhealthy(response.status_code):
+        if self.unhealthy(response.status_code) and not ignore_exception:
             raise KubeHTTPException(
                 response,
                 'restart Deployment "{}" in Namespace "{}"', name, namespace

From 23e70e8a91645c6276ceedce7043489088baec63 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 15 May 2024 12:20:34 +0800
Subject: [PATCH 006/200] chore(models): add procfile_type check

---
 ...type_alter_route_procfile_type_and_more.py | 28 ++++++++
 rootfs/api/models/app.py                      |  6 +-
 rootfs/api/models/base.py                     |  6 ++
 rootfs/api/models/domain.py                   |  4 +-
 rootfs/api/models/gateway.py                  |  6 +-
 rootfs/api/models/service.py                  |  4 +-
 rootfs/api/serializers/__init__.py            | 50 ++++++-------
 rootfs/api/tests/test_app.py                  |  3 +-
 rootfs/api/tests/test_build.py                |  3 +-
 .../api/tests/test_certificate_use_case_1.py  |  3 +-
 .../api/tests/test_certificate_use_case_2.py  |  3 +-
 .../api/tests/test_certificate_use_case_3.py  |  3 +-
 .../api/tests/test_certificate_use_case_4.py  |  3 +-
 .../api/tests/test_certificate_use_case_5.py  |  3 +-
 rootfs/api/tests/test_config.py               |  3 +-
 rootfs/api/tests/test_domain.py               |  2 +-
 rootfs/api/tests/test_gateway.py              |  3 +-
 rootfs/api/tests/test_procfile.py             | 70 +++++++++++++++++++
 rootfs/api/tests/test_release.py              |  3 +-
 rootfs/api/tests/test_tls.py                  |  3 +-
 20 files changed, 156 insertions(+), 53 deletions(-)
 create mode 100644 rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py
 create mode 100644 rootfs/api/tests/test_procfile.py

diff --git a/rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py b/rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py
new file mode 100644
index 000000000..6497eec76
--- /dev/null
+++ b/rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.10 on 2024-05-15 04:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0007_domain_procfile_type'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='domain',
+            name='procfile_type',
+            field=models.CharField(max_length=63),
+        ),
+        migrations.AlterField(
+            model_name='route',
+            name='procfile_type',
+            field=models.CharField(max_length=63),
+        ),
+        migrations.AlterField(
+            model_name='service',
+            name='procfile_type',
+            field=models.CharField(max_length=63),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index fa7253818..5c5664c02 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -25,7 +25,7 @@
 from api.utils import generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
-from .gateway import Gateway, Route, DEFAULT_HTTP_PORT
+from .gateway import Gateway, Route
 from .limit import LimitPlan
 from .config import Config
 from .service import Service
@@ -33,12 +33,10 @@
 from .tls import TLS
 from .appsettings import AppSettings
 from .volume import Volume
-from .base import UuidAuditedModel
+from .base import UuidAuditedModel, PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN, DEFAULT_HTTP_PORT
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
-PROCFILE_TYPE_WEB = "web"
-PROCFILE_TYPE_RUN = "run"
 
 
 # http://kubernetes.io/v1.1/docs/design/identifiers.html
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index b59563906..87ae5353a 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -27,6 +27,12 @@
         "access_token", "expires_in", "token_type", "scope", "refresh_token"
     ],
 }
+PROCFILE_TYPE_WEB = "web"
+PROCFILE_TYPE_RUN = "run"
+DEFAULT_HTTP_PORT = 80
+DEFAULT_HTTPS_PORT = 443
+PROCFILE_TYPE_MIN_LENGTH = 3
+PROCFILE_TYPE_MAX_LENGTH = 63
 
 
 def get_anonymous_user_instance(user): return user(id=-1, username=settings.ANONYMOUS_USER_NAME)
diff --git a/rootfs/api/models/domain.py b/rootfs/api/models/domain.py
index 1d2def5fc..a77952905 100644
--- a/rootfs/api/models/domain.py
+++ b/rootfs/api/models/domain.py
@@ -1,7 +1,7 @@
 from django.db import models
 from django.db import transaction
 from django.contrib.auth import get_user_model
-from .base import AuditedModel
+from .base import AuditedModel, PROCFILE_TYPE_MAX_LENGTH
 
 User = get_user_model()
 
@@ -21,7 +21,7 @@ class Domain(AuditedModel):
         blank=True,
         null=True
     )
-    procfile_type = models.TextField()
+    procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
 
     class Meta:
         ordering = ['domain', 'certificate']
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index cb5b643c2..defce49db 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -8,12 +8,10 @@
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 
-from .base import AuditedModel
+from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, PROCFILE_TYPE_MAX_LENGTH
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
-DEFAULT_HTTP_PORT = 80
-DEFAULT_HTTPS_PORT = 443
 
 TLS_PROTOCOLS = ("HTTPS", "TLS")
 HOSTNAME_PROTOCOLS = TLS_PROTOCOLS + ("HTTP", )
@@ -190,7 +188,7 @@ class Route(AuditedModel):
     rules = models.JSONField(default=list)
     routable = models.BooleanField(default=True)
     parent_refs = models.JSONField(default=list)
-    procfile_type = models.TextField()
+    procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
 
     @property
     def protocols(self):
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index c0a00f70e..3d44fa1b0 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -6,7 +6,7 @@
 from api.utils import validate_json
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
-from .base import AuditedModel
+from .base import AuditedModel, PROCFILE_TYPE_MAX_LENGTH
 
 
 User = get_user_model()
@@ -34,7 +34,7 @@ class Service(AuditedModel):
     ports = models.JSONField(
         default=list, validators=[partial(validate_json, schema=service_ports_schema)])
     canary = models.BooleanField(default=False)
-    procfile_type = models.TextField()
+    procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
 
     class Meta:
         get_latest_by = 'created'
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 9ec6e80f8..7aab01ed8 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -17,6 +17,7 @@
 from api import models
 from api.utils import validate_json
 from api.exceptions import DryccException
+from api.models.base import PROCFILE_TYPE_MIN_LENGTH, PROCFILE_TYPE_MAX_LENGTH
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 from .schemas.rules import SCHEMA as RULES_SCHEMA
 from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
@@ -69,6 +70,16 @@
 HEALTHCHECK_MISMATCH_MSG = "Healthcheck pattern: %s" % HEALTHCHECK_MATCH.pattern
 
 
+def validate_procfile_type(value):
+    if not re.match(PROCTYPE_MATCH, value):
+        raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+    if len(value) < PROCFILE_TYPE_MIN_LENGTH or len(value) > PROCFILE_TYPE_MAX_LENGTH:
+        raise serializers.ValidationError(
+            "The length of procfile_type must be between {} and {}".format(
+                PROCFILE_TYPE_MIN_LENGTH, PROCFILE_TYPE_MAX_LENGTH))
+    return value
+
+
 class JSONFieldSerializer(serializers.JSONField):
     def __init__(self, *args, **kwargs):
         self.convert_to_str = kwargs.pop('convert_to_str', True)
@@ -199,16 +210,17 @@ def validate_procfile(data):
         for key, value in data.items():
             if value is None or value == "":
                 raise serializers.ValidationError("Command can't be empty for process type")
-
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
+            validate_procfile_type(key)
         return data
 
     @staticmethod
     def validate_dryccfile(data):
         if data:
-            return validate_json(data, DRYCCFILE_SCHEMA, serializers.ValidationError)
+            validate_json(data, DRYCCFILE_SCHEMA, serializers.ValidationError)
+            procfile_types = set().union(data.get("deploy", {}).keys())
+            procfile_types = procfile_types.union(data.get("build", {}).get("docker", {}).keys())
+            for procfile_type in procfile_types:
+                validate_procfile_type(procfile_type)
         return data
 
 
@@ -460,12 +472,7 @@ def ToACE(x): return idna.alabel(x).decode("utf-8", "strict")
 
         return aceValue
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-        return value
+    validate_procfile_type = staticmethod(validate_procfile_type)
 
 
 class ServiceSerializer(serializers.ModelSerializer):
@@ -502,12 +509,7 @@ def validate_protocol(value):
     def validate_target_port(cls, value):
         return cls.validate_port(value)
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-        return value
+    validate_procfile_type = staticmethod(validate_procfile_type)
 
 
 class CertificateSerializer(serializers.ModelSerializer):
@@ -719,12 +721,7 @@ def validate_protocol(value):
             raise serializers.ValidationError(GATEWAY_PROTOCOL_MISMATCH_MSG)
         return value
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-        return value
+    validate_procfile_type = staticmethod(validate_procfile_type)
 
 
 class RouteSerializer(serializers.Serializer):
@@ -751,12 +748,7 @@ def validate_kind(value):
             raise serializers.ValidationError(ROUTE_PROTOCOL_MISMATCH_MSG)
         return value
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-        return value
+    validate_procfile_type = staticmethod(validate_procfile_type)
 
     @staticmethod
     def validate_rules(value):
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index e2a1a3dfb..80adf01dc 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -15,7 +15,8 @@
 from django.core.cache import cache
 from django.test.utils import override_settings
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.config import Config
 from scheduler import KubeException, KubeHTTPException
 
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 56b7f7f39..9bd4c027b 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -11,7 +11,8 @@
 from unittest import mock
 
 from api.models.build import Build
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from scheduler import KubeException
 
 from api.tests import adapter, DryccTransactionTestCase
diff --git a/rootfs/api/tests/test_certificate_use_case_1.py b/rootfs/api/tests/test_certificate_use_case_1.py
index a4511605f..1ec9a361f 100644
--- a/rootfs/api/tests/test_certificate_use_case_1.py
+++ b/rootfs/api/tests/test_certificate_use_case_1.py
@@ -1,7 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
diff --git a/rootfs/api/tests/test_certificate_use_case_2.py b/rootfs/api/tests/test_certificate_use_case_2.py
index 552026559..907bceb1c 100644
--- a/rootfs/api/tests/test_certificate_use_case_2.py
+++ b/rootfs/api/tests/test_certificate_use_case_2.py
@@ -1,7 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
diff --git a/rootfs/api/tests/test_certificate_use_case_3.py b/rootfs/api/tests/test_certificate_use_case_3.py
index c4dd0b890..4caff2aee 100644
--- a/rootfs/api/tests/test_certificate_use_case_3.py
+++ b/rootfs/api/tests/test_certificate_use_case_3.py
@@ -1,7 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
diff --git a/rootfs/api/tests/test_certificate_use_case_4.py b/rootfs/api/tests/test_certificate_use_case_4.py
index aa8802400..7f829d6fb 100644
--- a/rootfs/api/tests/test_certificate_use_case_4.py
+++ b/rootfs/api/tests/test_certificate_use_case_4.py
@@ -1,7 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
diff --git a/rootfs/api/tests/test_certificate_use_case_5.py b/rootfs/api/tests/test_certificate_use_case_5.py
index 5667a48d8..0a352241e 100644
--- a/rootfs/api/tests/test_certificate_use_case_5.py
+++ b/rootfs/api/tests/test_certificate_use_case_5.py
@@ -1,7 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index ecf562c9f..119cb64d9 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -13,7 +13,8 @@
 
 from unittest import mock
 
-from api.models.app import App, PROCFILE_TYPE_RUN, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_RUN, PROCFILE_TYPE_WEB
 from api.models.config import Config
 from api.serializers import CONFIG_LIMITS_MISMATCH_MSG
 from api.models.build import Build
diff --git a/rootfs/api/tests/test_domain.py b/rootfs/api/tests/test_domain.py
index 1faca2cfa..c771ce989 100644
--- a/rootfs/api/tests/test_domain.py
+++ b/rootfs/api/tests/test_domain.py
@@ -8,7 +8,7 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import PROCFILE_TYPE_WEB
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.domain import Domain
 from api.tests import DryccTestCase
 
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 4ea4edf7d..6efb8e44d 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -5,7 +5,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.build import Build
 from api.models.gateway import Route
 from api.models.release import Release
diff --git a/rootfs/api/tests/test_procfile.py b/rootfs/api/tests/test_procfile.py
new file mode 100644
index 000000000..2748b2181
--- /dev/null
+++ b/rootfs/api/tests/test_procfile.py
@@ -0,0 +1,70 @@
+from rest_framework.test import APITestCase
+from rest_framework import serializers
+from api.serializers import validate_procfile_type, BuildSerializer, ServiceSerializer
+
+
+class ProcfileTypeTest(APITestCase):
+
+    def assertException(self, method, exception=Exception, *args, **kwargs):
+        try:
+            method(*args, **kwargs)
+        except BaseException as e:
+            if not isinstance(e, exception):
+                raise AssertionError(
+                    "Did not throw the expected exception, "
+                    f"actual: {type(e)}, expect: {exception}"
+                )
+        else:
+            raise AssertionError(
+                f"Did not throw the expected exception, args: {args}, kwargs: {kwargs}")
+
+    def test_procfile_type_error(self):
+        for procfile_type in ["w", "we", "-a", "we-canary", "w" * 64]:
+            self.assertException(
+                validate_procfile_type, serializers.ValidationError, procfile_type)
+
+    def test_procfile_type_ok(self):
+        self.assertEqual(validate_procfile_type("web"), "web")
+        self.assertEqual(validate_procfile_type("w" * 63), "w" * 63)
+        self.assertEqual(validate_procfile_type("web-canary-be"), "web-canary-be")
+
+    def test_dryccfile_procfile_type(self):
+        dryccfile_1 = {
+            "build": {
+                "docker": {
+                    "web": "Dockerfile",
+                    "task": "Dockerfile.task"
+                },
+            },
+            "deploy": {
+                "web": {
+                    "args": ["python", "-m", "http.server", "5000"]
+                },
+                "task": {
+                    "command": ["sleep"],
+                    "args": ["infinity"]
+                },
+            }
+        }
+        validate_dryccfile = BuildSerializer().validate_dryccfile
+        self.assertEqual(validate_dryccfile(dryccfile_1), dryccfile_1)
+        dryccfile_1["build"]["docker"]["w"] = "Dockerfile.w"
+        self.assertException(
+                validate_dryccfile, serializers.ValidationError, dryccfile_1)
+        del dryccfile_1["build"]["docker"]["w"]
+        dryccfile_1["deploy"]["w"] = {"args": ["python", "-m", "http.server", "5000"]}
+        self.assertException(
+                validate_dryccfile, serializers.ValidationError, dryccfile_1)
+        del dryccfile_1["deploy"]["w"]
+        dryccfile_1["deploy"]["w-canary"] = {"args": ["python", "-m", "http.server", "5000"]}
+        self.assertException(
+                validate_dryccfile, serializers.ValidationError, dryccfile_1)
+
+    def test_staticmethod_procfile_type(self):
+        s_validate_procfile_type = ServiceSerializer().validate_procfile_type
+        for procfile_type in ["w", "we", "-a", "we-canary", "w" * 64]:
+            self.assertException(
+                s_validate_procfile_type, serializers.ValidationError, procfile_type)
+        self.assertEqual(s_validate_procfile_type("web"), "web")
+        self.assertEqual(s_validate_procfile_type("w" * 63), "w" * 63)
+        self.assertEqual(s_validate_procfile_type("web-canary-be"), "web-canary-be")
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 1dbcf572d..0d5c838a6 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -10,7 +10,8 @@
 from django.test.utils import override_settings
 from unittest import mock
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.build import Build
 from api.models.release import Release
 from scheduler import KubeHTTPException
diff --git a/rootfs/api/tests/test_tls.py b/rootfs/api/tests/test_tls.py
index 8e595313c..bbdcbeed9 100644
--- a/rootfs/api/tests/test_tls.py
+++ b/rootfs/api/tests/test_tls.py
@@ -3,7 +3,8 @@
 from django.core.cache import cache
 from django.contrib.auth import get_user_model
 
-from api.models.app import App, PROCFILE_TYPE_WEB
+from api.models.app import App
+from api.models.base import PROCFILE_TYPE_WEB
 from api.models.tls import TLS
 from api.tests import adapter, DryccTransactionTestCase
 

From f188a1fb8487eaf93cead49febabced2b5eb7534 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 15 May 2024 18:44:57 +0800
Subject: [PATCH 007/200] chore(config): change config key pattern

---
 rootfs/api/serializers/__init__.py | 15 +++------------
 rootfs/api/tests/test_config.py    |  2 +-
 2 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 7aab01ed8..aedfba500 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -39,19 +39,10 @@
     "the route kind only supports: %s" % ROUTE_PROTOCOL_MATCH.pattern)
 PROCTYPE_MATCH = re.compile(PROCTYPE_REGEX)
 PROCTYPE_MISMATCH_MSG = "Process types can only supports: %s" % PROCTYPE_MATCH.pattern
-MEMLIMIT_MATCH = re.compile(r'^(?P<mem>([1-9][0-9]*[mgMG]))$', re.IGNORECASE)
-MEMLIMIT_MISMATCH_MSG = (
-    "Memory limit format: <number><unit>, "
-    "where unit = M or G"
-)
-CPUSHARE_MATCH = re.compile(r'^(?P<cpu>([-+]?[1-9][0-9]*[m]?))$')
-CPUSHARE_MISMATCH_MSG = "CPU limit format: <value>, where value must be a numeric"
+
 TAGVAL_MATCH = re.compile(r'^(?:[a-zA-Z\d][-\.\w]{0,61})?[a-zA-Z\d]$')
-CONFIGKEY_MATCH = re.compile(r'^[a-z_]+[a-z0-9_]*$', re.IGNORECASE)
-CONFIGKEY_MISMATCH_MSG = (
-    "Config keys must start with a letter or underscore and "
-    "only contain [A-z0-9_]"
-)
+CONFIGKEY_MATCH = re.compile(r'^[A-z0-9_\-\.]+$', re.IGNORECASE)
+CONFIGKEY_MISMATCH_MSG = ("Config key can only supports: %s" % CONFIGKEY_MATCH.pattern)
 CONFIG_LIMITS_MISMATCH_MSG = "The limit plan {} does not exist"
 
 TERMINATION_GRACE_PERIOD_MATCH = re.compile(r'^[0-9]*$')
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 119cb64d9..f44d45405 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -317,7 +317,7 @@ def test_config_deploy_failure(self, mock_requests):
     def test_invalid_config_keys(self, mock_requests):
         """Test that invalid config keys are rejected.
         """
-        keys = ("123", "../../foo", "FOO/", "FOO-BAR")
+        keys = ("$123", "../../foo", "FOO/", "*FOO-BAR")
         app_id = self.create_app()
         url = f'/v2/apps/{app_id}/config'
         for k in keys:

From 3de12693d6576cd6e85ec05e66e372d488deab72 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 16 May 2024 18:36:23 +0800
Subject: [PATCH 008/200] chore(gunicorn): add keepalive

---
 rootfs/drycc/gunicorn/config.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rootfs/drycc/gunicorn/config.py b/rootfs/drycc/gunicorn/config.py
index 9927f2f40..7395214e8 100644
--- a/rootfs/drycc/gunicorn/config.py
+++ b/rootfs/drycc/gunicorn/config.py
@@ -21,10 +21,11 @@
 else:
     bind = '0.0.0.0:8000'
 
-workers = int(os.environ.get('GUNICORN_WORKERS', 2))
+workers = int(os.environ.get('GUNICORN_WORKERS', 4))
+timeout = int(os.environ.get('GUNICORN_TIMEOUT', 30))
+keepalive = int(os.environ.get('GUNICORN_KEEPALIVE', 60))
 worker_class = "uvicorn.workers.UvicornWorker"
 pythonpath = dirname(dirname(dirname(realpath(__file__))))
-timeout = 1200
 pidfile = '/tmp/gunicorn.pid'
 logger_class = 'drycc.gunicorn.logging.Logging'
 loglevel = 'info'

From 9ce6d9fc9fe1324b47044142884003c99c45c88d Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sun, 19 May 2024 18:48:57 +0800
Subject: [PATCH 009/200] chore(channels): add cache user permission

---
 rootfs/api/authentication.py      |  2 +-
 rootfs/api/backend.py             | 29 ++++++++++++++++++++---------
 rootfs/api/consumers.py           | 23 +++++++++++++----------
 rootfs/api/manager.py             | 23 ++++++++++++++++++++---
 rootfs/api/middleware.py          | 16 +++++++++-------
 rootfs/api/models/app.py          | 28 ----------------------------
 rootfs/api/models/certificate.py  | 22 +++++++++++++---------
 rootfs/api/models/gateway.py      |  2 +-
 rootfs/api/permissions.py         | 12 ++++++------
 rootfs/api/settings/production.py |  2 +-
 rootfs/api/tests/test_gateway.py  |  8 +++++---
 rootfs/api/views.py               | 21 +++++----------------
 rootfs/requirements.txt           |  4 ++--
 13 files changed, 96 insertions(+), 96 deletions(-)

diff --git a/rootfs/api/authentication.py b/rootfs/api/authentication.py
index d423172a5..61a9c1f7a 100644
--- a/rootfs/api/authentication.py
+++ b/rootfs/api/authentication.py
@@ -65,7 +65,7 @@ def authenticate_credentials(self, key):
             from api.backend import OauthCacheManager
             token.refresh_token()
             user = OauthCacheManager().get_user(token.oauth['access_token'])
-            cache.set(token, user, timeout=token.oauth['expires_in'])
+            cache.set(key, user, timeout=token.oauth['expires_in'])
             return user, token.key
         return (token.owner, token.key)
 
diff --git a/rootfs/api/backend.py b/rootfs/api/backend.py
index 4b8864279..bfa91e871 100644
--- a/rootfs/api/backend.py
+++ b/rootfs/api/backend.py
@@ -32,16 +32,27 @@ class DryccOIDC(OpenIdConnectAuth):
         ('scope', 'scope'),
     ]
 
+    def __init__(self, *args, **kwargs):
+        self.timeout = 3  # request timeout
+        super().__init__(*args, **kwargs)
+
     @social_cache(ttl=86400)
     def oidc_config(self):
-        return self.get_json(self.OIDC_ENDPOINT +
-                             '/.well-known/openid-configuration/')
+        return self.get_json(
+            self.OIDC_ENDPOINT + '/.well-known/openid-configuration/',
+            timeout=self.timeout
+        )
 
     def get_user_data(self, access_token):
         """Loads user data from service"""
         url = settings.SOCIAL_AUTH_DRYCC_USERINFO_URL
-        response = self.get_json(url, headers={
-            'authorization': 'Bearer ' + access_token})
+        response = self.get_json(
+            url,
+            headers={
+                'authorization': 'Bearer ' + access_token
+            },
+            timeout=self.timeout,
+        )
         return {
             'id': response.get('id'),
             'username': response.get('username'),
@@ -62,13 +73,13 @@ def refresh_token(self, refresh_token):
                 'client_id': settings.SOCIAL_AUTH_DRYCC_KEY,
                 'refresh_token': refresh_token,
             },
+            timeout=self.timeout,
         )
 
 
 class OauthCacheManager(object):
 
-    def __init__(self, timeout=60 * 10):
-        self.timeout = timeout
+    def __init__(self):
         self.drycc_oauth = DryccOIDC()
 
     def get_user(self, access_token):
@@ -82,13 +93,13 @@ def _get_user(access_token):
                 logger.info(e)
                 raise exceptions.AuthenticationFailed(gettext_lazy('Verify token fail.'))
         return cache.get_or_set(
-            access_token, lambda: _get_user(access_token), settings.OAUTH_CACHE_USER_TIME)
+            access_token, lambda: _get_user(access_token), settings.DRYCC_CACHE_USER_TIME)
 
     def set_state(self, key, state):
-        cache.set("oidc_key_" + key, state, self.timeout)
+        cache.set("oidc_key_" + key, state, settings.DRYCC_CACHE_USER_TIME)
 
     def set_token(self, state, data):
-        cache.set("oidc_state_" + state, data, self.timeout)
+        cache.set("oidc_state_" + state, data, settings.DRYCC_CACHE_USER_TIME)
 
     def get_token(self, key):
         state = cache.get("oidc_key_" + key, "")
diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index 10a3dadf8..bdf3982d2 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -4,8 +4,8 @@
 import ssl
 import aiohttp
 import asyncio
-import collections
 from django.conf import settings
+from django.core.cache import cache
 
 from asgiref.sync import sync_to_async, async_to_sync
 
@@ -22,21 +22,24 @@
 from .permissions import has_app_permission
 
 
-Request = collections.namedtuple("Request", ["user", "method"])
-
-
 class BaseAppConsumer(AsyncWebsocketConsumer):
+    timeout = 60 * 60
 
     @database_sync_to_async
     def has_perm(self):
         if self.scope["user"] is None:
             return False, "user not login"
-        request = Request(self.scope["user"], "POST")
-        try:
-            app = App.objects.get(id=self.id)
-            return has_app_permission(request, app)
-        except App.DoesNotExist:
-            return False, "user not exists"
+        key = f"permission:user:{self.scope["user"].id}:app:{self.id}"
+        permission = cache.get(key)
+        if permission is None:
+            try:
+                app = App.objects.get(id=self.id)
+                permission = has_app_permission(self.scope["user"], app, "GET")
+                if permission[0]:
+                    cache.set(key, permission, timeout=self.timeout)
+            except App.DoesNotExist:
+                permission = (False, "user not exists")
+        return permission
 
     async def connect(self):
         self.id = self.scope["url_route"]["kwargs"]["id"]
diff --git a/rootfs/api/manager.py b/rootfs/api/manager.py
index 18a7b2ebb..5b985aabd 100644
--- a/rootfs/api/manager.py
+++ b/rootfs/api/manager.py
@@ -1,14 +1,19 @@
 import base64
+import logging
 import requests
 from typing import List, Dict
+from django.core.cache import cache
 from requests_toolbelt import user_agent
 from django.conf import settings
 from api import __version__ as drycc_version
 
+logger = logging.getLogger(__name__)
+
 
 class ManagerAPI(object):
 
-    def __init__(self):
+    def __init__(self, timeout=3):
+        self.timeout = timeout
         token = base64.b85encode(b"%s:%s" % (
             settings.WORKFLOW_MANAGER_ACCESS_KEY.encode("utf8"),
             settings.WORKFLOW_MANAGER_SECRET_KEY.encode("utf8"),
@@ -23,6 +28,7 @@ def request(self, method, url, **kwargs):
         headers = kwargs.get("headers", {})
         headers.update(self.headers)
         kwargs["headers"] = headers
+        kwargs["timeout"] = self.timeout
         return requests.request(method, url, **kwargs)
 
     def get(self, url, params=None, **kwargs):
@@ -57,8 +63,19 @@ def get_status(self, id):
             "message": "The user is in arrears"
         }
         """
-        url = f"{settings.WORKFLOW_MANAGER_URL}/users/{id}/status/"
-        return self.get(url=url).json()
+        key = f"user:status:{id}"
+        status = cache.get(key)
+        if not status:
+            url = f"{settings.WORKFLOW_MANAGER_URL}/users/{id}/status/"
+            try:
+                status = self.get(url=url, timeout=self.timeout).json()
+            except requests.exceptions.Timeout as ex:
+                msg = f"request user {id} timeout, skipping verification."
+                status = {"is_active": True, "message": msg}
+                logger.error(msg)
+                logger.exception(ex)
+            cache.set(key, status, timeout=settings.DRYCC_CACHE_USER_TIME)
+        return status
 
 
 class Measurement(ManagerAPI):
diff --git a/rootfs/api/middleware.py b/rootfs/api/middleware.py
index 651f60982..e897443d5 100644
--- a/rootfs/api/middleware.py
+++ b/rootfs/api/middleware.py
@@ -3,8 +3,8 @@
 
 See https://docs.djangoproject.com/en/1.11/topics/http/middleware/
 """
-import collections
 from api import __version__
+from django.http.request import HttpRequest
 
 from channels.middleware import BaseMiddleware
 from channels.db import database_sync_to_async
@@ -36,18 +36,20 @@ class ChannelOAuthMiddleware(BaseMiddleware):
     """
     Middleware which populates scope["user"] from a auth2 token.
     """
-    Request = collections.namedtuple('Request', ["META"])
+    authentication = DryccAuthentication()
 
-    async def get_user(self, scope):
+    @database_sync_to_async
+    def get_user(self, scope):
         headers = {}
         for header in scope["headers"]:
-            if header[0] in (b"user-agent", b"authorization"):
+            if header[0] in (b"authorization", ):
                 key = "HTTP_%s" % header[0].decode().replace("-", "_").upper()
                 headers[key] = header[1].decode()
-        if len(headers) != 2:
+        if len(headers) < 1:
             return None
-        request = self.Request(headers)
-        user, _ = await database_sync_to_async(DryccAuthentication().authenticate)(request)
+        request = HttpRequest()
+        request.META = headers
+        user, _ = self.authentication.authenticate(request)
         return user
 
     async def __call__(self, scope, receive, send):
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 5c5664c02..512917420 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1,4 +1,3 @@
-import backoff
 import base64
 import functools
 import json
@@ -341,33 +340,6 @@ def cleanup_old(self, procfile_types=None):
                     self.scheduler().deployments.delete(self.id, name, True)
         self.log(f"cleanup old kubernetes deployments for {self.id}")
 
-    @backoff.on_exception(backoff.expo, ServiceUnavailable, max_tries=3)
-    def logs(self, log_lines=str(settings.LOG_LINES)):
-        """Return aggregated log data for this application."""
-        url = "http://{}:{}/logs/{}?log_lines={}".format(
-            settings.LOGGER_HOST, settings.LOGGER_PORT, self.id, log_lines)
-        try:
-            r = requests.get(url)
-        # Handle HTTP request errors
-        except requests.exceptions.RequestException as e:
-            msg = "Error accessing drycc-logger using url '{}': {}".format(url, e)
-            logger.error(msg)
-            raise ServiceUnavailable(msg) from e
-
-        # Handle logs empty or not found
-        if r.status_code == 204 or r.status_code == 404:
-            logger.info("GET {} returned a {} status code".format(url, r.status_code))
-            raise NotFound('Could not locate logs')
-
-        # Handle unanticipated status codes
-        if r.status_code != 200:
-            logger.error("Error accessing drycc-logger: GET {} returned a {} status code"
-                         .format(url, r.status_code))
-            raise ServiceUnavailable('Error accessing drycc-logger')
-
-        # cast content to string since it comes as bytes via the requests object
-        return str(r.content.decode('utf-8'))
-
     def run(self, user, image=None, command=None, args=None, volumes=None,
             timeout=3600, expires=3600, **kwargs):
         def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index 1e1d862a3..45c473a99 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -130,6 +130,10 @@ def domains(self):
 
         return domains
 
+    @property
+    def certname(self):
+        return '%s-certificate' % self.name
+
     def __str__(self):
         return self.name
 
@@ -197,24 +201,23 @@ def attach_in_kubernetes(self, domain):
         """Creates the certificate as a kubernetes secret"""
         # only create if it exists - We raise an exception when a secret doesn't exist
         try:
-            name = '%s-certificate' % self.name
             namespace = domain.app.id
             data = {
                 'tls.crt': self.certificate,
                 'tls.key': self.key
             }
 
-            secret = self.scheduler().secret.get(namespace, name).json()['data']
+            secret = self.scheduler().secret.get(namespace, self.certname).json()['data']
         except KubeException:
-            self.scheduler().secret.create(namespace, name, data)
+            self.scheduler().secret.create(namespace, self.certname, data)
         else:
             # update cert secret to the TLS Ingress format if required
             if secret != data:
                 try:
-                    self.scheduler().secret.update(namespace, name, data)
+                    self.scheduler().secret.update(namespace, self.certname, data)
                 except KubeException as e:
                     msg = 'There was a problem updating the certificate secret ' \
-                          '{} for {}'.format(name, namespace)
+                          '{} for {}'.format(self.certname, namespace)
                     raise ServiceUnavailable(msg) from e
 
     def detach(self, *args, **kwargs):
@@ -223,14 +226,15 @@ def detach(self, *args, **kwargs):
         domain.certificate = None
         domain.save()
 
-        name = '%s-certificate' % self.name
         namespace = domain.app.id
 
         # only delete if it exists and if no other domains depend on secret
         if len(self.domains) == 0:
             try:
                 # We raise an exception when a secret doesn't exist
-                self.scheduler().secret.get(namespace, name)
-                self.scheduler().secret.delete(namespace, name)
+                self.scheduler().secret.get(namespace, self.certname)
+                self.scheduler().secret.delete(namespace, self.certname)
             except KubeException as e:
-                raise ServiceUnavailable("Could not delete certificate secret {} for application {}".format(name, namespace)) from e  # noqa
+                raise ServiceUnavailable(
+                    "Could not delete certificate secret {} for application {}".format(
+                        self.certname, namespace)) from e
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index defce49db..b35173e42 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -68,7 +68,7 @@ def listeners(self):
                         "protocol": protocol,
                     }
                     secret_name = f"{self.app.id}-auto-tls" if auto_tls else (
-                        domain.certificate.name if domain.certificate else None)
+                        domain.certificate.certname if domain.certificate else None)
                     if secret_name and protocol in TLS_PROTOCOLS:
                         listener["tls"] = {
                             "certificateRefs": [{"kind": "Secret", "name": secret_name}]}
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index 4d652324d..f7fcab261 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -17,17 +17,17 @@ def get_app_status(app):
     return True, None
 
 
-def has_app_permission(request, obj):
+def has_app_permission(user, obj, method):
     if isinstance(obj, App) or hasattr(obj, 'app'):
         app = obj if isinstance(obj, App) else obj.app
         is_ok, message = get_app_status(app)
         if is_ok:
-            if request.user.is_superuser:
+            if user.is_superuser:
                 return True, None
-            elif app.owner == request.user:
+            elif app.owner == user:
                 return True, None
-            elif request.user.is_staff or request.user.has_perm('use_app', app):
-                if request.method != 'DELETE':
+            elif user.is_staff or user.has_perm('use_app', app):
+                if method != 'DELETE':
                     return True, None
                 else:
                     return False, "User does not have permission to delete"
@@ -81,7 +81,7 @@ class IsAppUser(permissions.BasePermission):
     an app-related model.
     """
     def has_object_permission(self, request, view, obj):
-        return has_app_permission(request, obj)[0]
+        return has_app_permission(request.user, obj, request.method)[0]
 
 
 class IsAdmin(permissions.BasePermission):
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 091e1aab0..22910641e 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -457,7 +457,7 @@
     'social_core.pipeline.user.user_details',
 )
 AUTHENTICATION_BACKENDS = ("api.backend.DryccOIDC", ) + AUTHENTICATION_BACKENDS
-OAUTH_CACHE_USER_TIME = int(os.environ.get('OAUTH_CACHE_USER_TIME', 30 * 60))
+DRYCC_CACHE_USER_TIME = int(os.environ.get('DRYCC_CACHE_USER_TIME', 30 * 60))
 
 # Redis Configuration
 DRYCC_REDIS_ADDRS = os.environ.get('DRYCC_REDIS_ADDRS', '127.0.0.1:6379').split(",")
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 6efb8e44d..c96aec747 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -180,7 +180,8 @@ def test_add_tls_listener(self):
                 'allowedRoutes': {'namespaces': {'from': 'All'}},
                 'name': 'tcp-443-1', 'port': 443, 'hostname': domain,
                 'protocol': 'TLS',
-                'tls': {'certificateRefs': [{'kind': 'Secret', 'name': secret_name}]}
+                'tls': {'certificateRefs': [
+                    {'kind': 'Secret', 'name': f"{secret_name}-certificate"}]}
             }],
             'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
         }]
@@ -207,7 +208,7 @@ def test_add_https_listener(self):
                 'tls': {
                     'certificateRefs': [{
                         'kind': 'Secret',
-                        'name': secret_name
+                        'name': f"{secret_name}-certificate"
                     }]
                 }
             }],
@@ -244,7 +245,8 @@ def test_add_http_listener(self):
                     'port': 443,
                     'hostname': domain,
                     'protocol': 'HTTPS',
-                    'tls': {'certificateRefs': [{'kind': 'Secret', 'name': secret_name}]}
+                    'tls': {'certificateRefs': [
+                        {'kind': 'Secret', 'name': f"{secret_name}-certificate"}]}
                 }
             ],
             'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 5fde0dc0e..86aaa1a2d 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -311,19 +311,6 @@ def scale(self, request, **kwargs):
         scale_app.delay(self.get_object(), request.user, request.data)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def logs(self, request, **kwargs):
-        app = self.get_object()
-        try:
-            logs = app.logs(request.query_params.get('log_lines', str(settings.LOG_LINES)))
-            return HttpResponse(logs, status=status.HTTP_200_OK, content_type='text/plain')
-        except NotFound:
-            return HttpResponse(status=status.HTTP_204_NO_CONTENT)
-        except ServiceUnavailable:
-            # TODO make 503
-            return HttpResponse("Error accessing logs for {}".format(app.id),
-                                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                                content_type='text/plain')
-
     def run(self, request, **kwargs):
         app = self.get_object()
         command = request.data.get('command', '').split()
@@ -691,7 +678,8 @@ def users(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
         request.user = get_object_or_404(User, username=kwargs['username'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request, app)
+        has_permission, message = permissions.has_app_permission(
+            request.user, app, request.method)
         if not has_permission:
             raise PermissionDenied(message)
 
@@ -721,7 +709,7 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         self.user = request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request, app)
+        has_permission, message = permissions.has_app_permission(self.user, app, request.method)
         if not has_permission:
             raise PermissionDenied(message)
         request.data['app'] = app
@@ -748,7 +736,8 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request, app)
+        has_permission, message = permissions.has_app_permission(
+            request.user, app, request.method)
         if not has_permission:
             raise PermissionDenied(message)
         config = app.release_set.filter(failed=False).latest().config
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index b2a509996..ec745c815 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -2,7 +2,7 @@
 inotify==0.2.10
 backoff==2.2.1
 django==4.2.11
-channels==4.0.0
+channels==4.1.0
 aiohttp==3.9.4
 django-cors-headers==4.3.1
 django-guardian==2.4.0
@@ -10,7 +10,7 @@ djangorestframework==3.15.1
 docker==7.0.0
 gunicorn==22.0.0
 uvicorn[standard]==0.29.0
-asgiref==3.8.0
+asgiref==3.8.1
 idna==3.7
 jsonschema==4.21.1
 morph==0.1.5

From 8b84add507f64138d20d8b4a974f33bd79b430d4 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 21 May 2024 15:27:43 +0800
Subject: [PATCH 010/200] fix(cert-manager): disabled auto tls error

---
 rootfs/api/models/gateway.py          | 37 ++++++++++++---------------
 rootfs/scheduler/resources/gateway.py |  4 +--
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index b35173e42..d71a5592e 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -51,10 +51,18 @@ def remove(self, port, protocol):
         self.ports = ports
         return True, ""
 
+    @property
+    def addresses(self):
+        data = self.scheduler().gateways.get(self.app.id, self.name, ignore_exception=True)
+        if data.status_code != 200:
+            return []
+        addresses = data.json()["status"].get("addresses", [])
+        return addresses
+
     @property
     def listeners(self):
-        listeners = []
         auto_tls = self.app.tls_set.latest().certs_auto_enabled
+        listeners = []
         domains = list(self._get_tls_domain(auto_tls))
         for item in self.ports:
             port, protocol = item["port"], item["protocol"]
@@ -82,34 +90,23 @@ def listeners(self):
                 })
         return listeners
 
-    @property
-    def addresses(self):
-        data = self.scheduler().gateways.get(self.app.id, self.name, ignore_exception=True)
-        if data.status_code != 200:
-            return []
-        addresses = data.json()["status"].get("addresses", [])
-        return addresses
-
     def refresh_to_k8s(self):
+        kwargs = {"listeners": self.listeners, "gateway_class": settings.GATEWAY_CLASS}
+        if self.app.tls_set.latest().certs_auto_enabled:
+            kwargs["annotations"] = {"cert-manager.io/issuer": self.app.id}
         try:
             try:
                 data = self.scheduler().gateways.get(self.app.id, self.name).json()
-                if len(self.listeners) > 0:
-                    self.scheduler().gateways.patch(self.app.id, self.name, **{
-                        "listeners": self.listeners,
-                        "gateway_class": settings.GATEWAY_CLASS,
-                        "version": data["metadata"]["resourceVersion"],
-                    })
+                if len(kwargs["listeners"]) > 0:
+                    kwargs["version"] = data["metadata"]["resourceVersion"]
+                    self.scheduler().gateways.patch(self.app.id, self.name, **kwargs)
                 else:
                     logger.debug("delete k8s resource when listeners are empty")
                     self.scheduler().gateways.delete(
                         self.app.id, self.name, ignore_exception=True)
             except KubeException:
-                if len(self.listeners) > 0:
-                    self.scheduler().gateways.create(self.app.id, self.name, **{
-                        "listeners": self.listeners,
-                        "gateway_class": settings.GATEWAY_CLASS,
-                    })
+                if len(kwargs["listeners"]) > 0:
+                    self.scheduler().gateways.create(self.app.id, self.name, **kwargs)
                 else:
                     logger.debug("skip creating k8s resource when listeners are empty")
         except KubeException as e:
diff --git a/rootfs/scheduler/resources/gateway.py b/rootfs/scheduler/resources/gateway.py
index 7f4710dd1..931bd8c86 100644
--- a/rootfs/scheduler/resources/gateway.py
+++ b/rootfs/scheduler/resources/gateway.py
@@ -13,9 +13,7 @@ def manifest(self, namespace, name, **kwargs):
             "metadata": {
                 "name": name,
                 "namespace": namespace,
-                "annotations": {
-                    "cert-manager.io/issuer": namespace
-                },
+                "annotations": kwargs.get("annotations", {}),
             },
             "spec": {
                 "gatewayClassName": kwargs.get("gateway_class", "default"),

From 1361a2cbd57beea64ead15c346512da980a6d1a7 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 22 May 2024 11:00:39 +0800
Subject: [PATCH 011/200] fix(channels): k8s api blocking

---
 rootfs/api/consumers.py | 157 +++++++++++++++++++++++++++-------------
 rootfs/requirements.txt |   2 +-
 2 files changed, 107 insertions(+), 52 deletions(-)

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index bdf3982d2..92d11ae89 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -7,7 +7,7 @@
 from django.conf import settings
 from django.core.cache import cache
 
-from asgiref.sync import sync_to_async, async_to_sync
+from asgiref.sync import sync_to_async
 
 from kubernetes.client import Configuration, exceptions
 from kubernetes.client.api import core_v1_api
@@ -67,58 +67,107 @@ def kubernetes(self):
 
 class AppLogsConsumer(BaseAppConsumer):
 
-    async def receive(self, text_data=None, bytes_data=None):
-        if text_data is not None:
-            kwargs = json.loads(text_data)
-            lines = kwargs.get("lines", 100)
-            follow = kwargs.get("follow", False)
-            timeout = kwargs.get("timeout", 300)
-            url = "http://{}:{}/logs/{}?log_lines={}&follow={}&timeout={}".format(
-                settings.LOGGER_HOST,
-                settings.LOGGER_PORT,
-                self.id,
-                lines,
-                follow,
-                timeout,
-            )
+    async def connect(self):
+        await super().connect()
+        self.session = None
+        self.running = False
+        self.conneted = True
+
+    async def task(self, **kwargs):
+        lines = kwargs.get("lines", 100)
+        follow = kwargs.get("follow", False)
+        timeout = kwargs.get("timeout", 300)
+        url = "http://{}:{}/logs/{}?log_lines={}&follow={}&timeout={}".format(
+            settings.LOGGER_HOST, settings.LOGGER_PORT, self.id, lines, follow, timeout,
+        )
+        try:
             async with aiohttp.ClientSession() as session:
+                self.session = session
                 async with session.get(url) as response:
                     async for data in response.content.iter_any():
+                        if not self.conneted:
+                            break
                         await self.send(text_data=data)
+        except asyncio.TimeoutError:
+            pass
+        finally:
             await self.close(code=1000)
-        else:
-            raise ValueError("text_data cannot be empty!")
+
+    async def receive(self, text_data=None, bytes_data=None):
+        if self.running:
+            return
+        self.running = True
+        kwargs = json.loads(text_data)
+        asyncio.create_task(self.task(**kwargs))
+
+    async def disconnect(self, close_code):
+        if self.session:
+            await self.session.close()
+        self.conneted = False
 
 
 class AppPodLogsConsumer(BaseK8sConsumer):
 
     async def connect(self):
         await super().connect()
+        self.running = False
+        self.response = None
+        self.conneted = True
         self.pod_id = self.scope["url_route"]["kwargs"]["pod_id"]
 
-    @sync_to_async
-    def receive(self, text_data=None, bytes_data=None):
-        kwargs = json.loads(text_data)
+    def reader(self, sock):
         try:
-            stream = self.kubernetes.read_namespaced_pod_log(self.pod_id, self.id, **{
-                "tail_lines": kwargs.get("lines", 100),
-                "follow": kwargs.get("follow", False),
-                "container": kwargs.get("container", ""),
-                "_preload_content": False,
-            }).stream()
-            for line in stream:
-                async_to_sync(self.send)(text_data=line)
-        except exceptions.ApiException as e:
-            async_to_sync(self.send)(text_data=str(e))
-        async_to_sync(self.close)(code=1000)
+            delimiter, buffer = b"\r\n", sock.read()
+            if delimiter in buffer:
+                index = buffer.index(delimiter)
+                length = int(buffer[:index], base=16)
+                if len(buffer) - (index + len(delimiter)) < length:
+                    buffer += sock.read()
+            while buffer and self.conneted:
+                index = buffer.index(delimiter)
+                length = int(buffer[:index], base=16)
+                if length == 0:
+                    asyncio.create_task(self.close(code=1000))
+                    break
+                start_pos = index + len(delimiter)
+                end_pos = start_pos + length + len(delimiter)
+                asyncio.create_task(
+                    self.send(bytes_data=buffer[start_pos:end_pos].strip(delimiter)))
+                buffer = buffer[end_pos:]
+        except BaseException:
+            asyncio.create_task(self.close(code=1000))
+
+    async def receive(self, text_data=None, bytes_data=None):
+        if self.running:
+            return
+        self.running = True
+        data = json.loads(text_data)
+        args = (self.pod_id, self.id)
+        kwargs = {
+            "tail_lines": data.get("lines", 100),
+            "follow": data.get("follow", False),
+            "container": data.get("container", ""),
+            "_preload_content": False,
+        }
+        loop = asyncio.get_event_loop()
+        self.response = await sync_to_async(self.kubernetes.read_namespaced_pod_log)(
+            *args, **kwargs)
+        loop.add_reader(self.response.connection.sock, self.reader, self.response.connection.sock)
+
+    async def disconnect(self, close_code):
+        if self.response:
+            loop = asyncio.get_event_loop()
+            loop.remove_reader(self.response.connection.sock)
+            await sync_to_async(self.response.close)()
+        self.conneted = False
 
 
 class AppPodExecConsumer(BaseK8sConsumer):
 
     async def connect(self):
+        await super().connect()
         self.stream = None
         self.conneted = True
-        await super().connect()
         self.pod_id = self.scope["url_route"]["kwargs"]["pod_id"]
 
     async def send(self, data, channel=STDOUT_CHANNEL):
@@ -131,22 +180,33 @@ async def send(self, data, channel=STDOUT_CHANNEL):
         elif isinstance(data, str):
             await super().send(text_data=channel_prefix+data)
 
+    async def wait(self):
+        future, loop = asyncio.Future(), asyncio.get_event_loop()
+        loop.add_reader(self.stream.sock, future.set_result, None)
+        future.add_done_callback(lambda f: loop.remove_reader(self.stream.sock))
+        await future
+
     async def task(self):
-        deadline = time.time() + settings.DRYCC_APP_POD_EXEC_TIMEOUT
-        while self.stream.is_open() and self.conneted and time.time() < deadline:
-            try:
-                await sync_to_async(self.stream.update)(0.1)
-                for channel in (ERROR_CHANNEL, STDOUT_CHANNEL, STDERR_CHANNEL):
-                    if channel in self.stream._channels:
-                        data = self.stream.read_channel(channel)
-                        await self.send(data, channel)
-            except ssl.SSLEOFError:
-                break
-        await self.close(code=1000)
+        try:
+            deadline = time.time() + settings.DRYCC_APP_POD_EXEC_TIMEOUT
+            while self.stream.is_open() and self.conneted and time.time() < deadline:
+                try:
+                    await self.wait()
+                    self.stream.update()
+                    for channel in (ERROR_CHANNEL, STDOUT_CHANNEL, STDERR_CHANNEL):
+                        if channel in self.stream._channels:
+                            data = self.stream.read_channel(channel)
+                            await self.send(data, channel)
+                except ssl.SSLEOFError:
+                    break
+        except exceptions.ApiException as e:
+            await self.send(str(e), STDERR_CHANNEL)
+        finally:
+            await self.close(code=1000)
 
     async def disconnect(self, close_code):
         if self.stream:
-            self.stream.close()
+            await sync_to_async(self.stream.close)()
         self.conneted = False
 
     async def receive(self, text_data=None, bytes_data=None):
@@ -154,13 +214,8 @@ async def receive(self, text_data=None, bytes_data=None):
             args = (self.kubernetes.connect_get_namespaced_pod_exec, self.pod_id, self.id)
             kwargs = json.loads(text_data)
             kwargs.update({"stderr": True, "stdout": True, "_preload_content": False})
-            try:
-                self.stream = stream(*args, **kwargs)
-            except exceptions.ApiException as e:
-                await self.send(str(e), STDERR_CHANNEL)
-                await self.close(code=1000)
-            else:
-                asyncio.create_task(self.task())
+            self.stream = await sync_to_async(stream)(*args, **kwargs)
+            asyncio.create_task(self.task())
         elif self.stream is not None:
             data = text_data if text_data else bytes_data
             channel, data = ord(data[0]), data[1:]
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index ec745c815..ff5f4e6b6 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -20,7 +20,7 @@ psycopg[binary]==3.1.18
 pyOpenSSL==24.1.0
 ndg-httpsclient==0.5.1
 pytz==2024.1
-requests==2.31.0
+requests==2.32.0
 requests-toolbelt==1.0.0
 celery==5.3.6
 hiredis==2.3.2

From c931926468896e23069c8e5c4af844ec47159811 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 22 May 2024 15:48:21 +0800
Subject: [PATCH 012/200] feat(pods): add describe pods

---
 rootfs/api/models/app.py       | 29 +++++++++++++++++++++++++++++
 rootfs/api/tests/test_build.py | 32 ++++++++++++++++----------------
 rootfs/api/tests/test_hooks.py |  4 ++--
 rootfs/api/tests/test_pods.py  | 26 ++++++--------------------
 rootfs/api/urls.py             | 11 ++++-------
 rootfs/api/views.py            |  9 +++++++++
 rootfs/scheduler/mock.py       | 13 ++++++++++++-
 7 files changed, 78 insertions(+), 46 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 512917420..ed3974c1c 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -385,6 +385,35 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             raise ServiceUnavailable(err) from e
         return name
 
+    def describe_pod(self, pod_name):
+        def get_commands_and_args(pod, container_name):
+            commands, args = [], []
+            for container in pod["spec"]["containers"]:
+                if container["name"] == container_name:
+                    args = container.get("args", [])
+                    commands = container.get("commands", [])
+                    break
+            return commands, args
+        result = []
+        try:
+            pod = self.scheduler().pod.get(self.id, pod_name).json()
+            for status in pod["status"]["containerStatuses"]:
+                commands, args = get_commands_and_args(pod, status["name"])
+                result.append({
+                    "container": status["name"],
+                    "image": status["image"],
+                    "commands": commands,
+                    "args": args,
+                    "state": status["state"],
+                    "lastState": status["lastState"],
+                    "ready": status["ready"],
+                    "restartCount": status["restartCount"],
+                })
+        except KubeHTTPException as e:
+            if e.response.status_code != 404:
+                raise e
+        return result
+
     def list_pods(self, *args, **kwargs):
         """Used to list basic information about pods running for a given application"""
         try:
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 9bd4c027b..baa7ba388 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -113,7 +113,7 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -132,7 +132,7 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -150,7 +150,7 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -171,7 +171,7 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -192,7 +192,7 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -238,7 +238,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # verify web
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -250,7 +250,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # verify worker
-        url = f"/v2/apps/{app_id}/pods/worker"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
@@ -263,13 +263,13 @@ def test_build_forgotten_procfile(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # verify worker is not there
-        url = f"/v2/apps/{app_id}/pods/worker"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
         # verify web is not there
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
@@ -311,7 +311,7 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # verify worker
-        url = f"/v2/apps/{app_id}/pods/worker"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
@@ -323,13 +323,13 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # verify worker is still there
-        url = f"/v2/apps/{app_id}/pods/worker"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'worker', "v3", "up")
 
         # verify web is still there
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v3", "up")
@@ -347,7 +347,7 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # verify worker info
-        url = f"/v2/apps/{app_id}/pods/worker"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'worker', "v3", "up")
@@ -383,7 +383,7 @@ def test_build_forgotten_procfile_reject(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # verify worker
-        url = f"/v2/apps/{app_id}/pods/worker"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
@@ -467,7 +467,7 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
@@ -491,7 +491,7 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index 389ec6e6d..432c46a99 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -230,7 +230,7 @@ def test_build_hook_procfile(self, mock_requests):
         self.assertEqual(build['procfile'], PROCFILE)
 
         # test listing/retrieving container info
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', 'v2')
@@ -270,7 +270,7 @@ def test_build_hook_dockerfile(self, mock_requests):
         self.assertEqual(build['sha'], SHA)
         self.assertEqual(build['dockerfile'], DOCKERFILE)
         # test default container
-        url = f"/v2/apps/{app_id}/pods/web"
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, 'web', 'v2')
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index ba823fd6b..a846b6bdf 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -82,20 +82,6 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(response.data['structure']['web'], 4)
         self.assertEqual(response.data['structure']['worker'], 2)
 
-        # test listing/retrieving container info
-        url = f"/v2/apps/{app_id}/pods/web"
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 4)
-        self.assertEqual(len(response.data['results']), 4)
-
-        name = response.data['results'][0]['name']
-        url = f"/v2/apps/{app_id}/pods/web/{name}"
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 1)
-        self.assertEqual(response.data['results'][0]['name'], name)
-
         # scale down
         url = f"/v2/apps/{app_id}/scale"
         # test setting two proc types at a time
@@ -164,12 +150,6 @@ def test_container_api(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
-        # test listing/retrieving container info
-        url = f"/v2/apps/{app_id}/pods/web"
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 6)
-
         # scale down
         url = f"/v2/apps/{app_id}/scale"
         body = {'web': 3}
@@ -263,6 +243,12 @@ def test_release(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, "web", 'v4')
+        pod_name = response.data['results'][0]["name"]
+        response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
+        self.assertEqual(response.status_code, 200, response.data)
+        pod_name = "no-exists-pod-name"
+        response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
+        self.assertEqual(response.status_code, 404, response.data)
 
     def test_container_errors(self, mock_requests):
         app_id = self.create_app()
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index ae83bd7b1..c5b8ffc07 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -53,16 +53,13 @@
         r"^apps/(?P<id>{})/pods/(?P<type>[-_\w.]+)/restart/?$".format(settings.APP_URL_REGEX),
         views.PodViewSet.as_view({'post': 'restart'})),
     # list pods
-    re_path(
-        r"^apps/(?P<id>{})/pods/(?P<type>[-_\w]+)/(?P<name>[-_\w]+)/?$".format(
-            settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list'})),
-    re_path(
-        r"^apps/(?P<id>{})/pods/(?P<type>[-_\w.]+)/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list'})),
     re_path(
         r"^apps/(?P<id>{})/pods/?$".format(settings.APP_URL_REGEX),
         views.PodViewSet.as_view({'get': 'list'})),
+    # describe pod
+    re_path(
+        r"^apps/(?P<id>{})/pods/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),
+        views.PodViewSet.as_view({'get': 'describe'})),
     # application domains
     re_path(
         r"^apps/(?P<id>{})/domains/(?P<domain>\**\.?[-\._\w]+)/?$".format(settings.APP_URL_REGEX),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 86aaa1a2d..f250d455b 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -441,6 +441,15 @@ def restart(self, *args, **kwargs):
         restart_app.delay(self.get_app(), **kwargs)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    def describe(self, *args, **kwargs):
+        pod_name = kwargs["name"]
+        data = self.get_app().describe_pod(pod_name)
+        if len(data) == 0:
+            return Response(status=status.HTTP_404_NOT_FOUND)
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
 
 class AppSettingsViewSet(AppResourceViewSet):
     model = models.appsettings.AppSettings
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index c7f779622..4dfae673c 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -368,6 +368,7 @@ def create_pods(url, labels, base, new_pods):
             'containerStatuses': [
                 {
                     'name': '{}-{}'.format(labels['app'], labels['type']),
+                    'image': '127.0.0.1/{}-{}'.format(labels['app'], labels['type']),
                     # TODO ready can be True / False (boolean)
                     'ready': True,
                     # TODO can be running / terminated / waiting
@@ -375,7 +376,17 @@ def create_pods(url, labels, base, new_pods):
                         'running': {
                             'startedAt': timestamp
                         }
-                    }
+                    },
+                    'lastState': {
+                        "terminated": {
+                            "containerID": f"cri-o://{uuid.uuid4().hex * 2}",
+                            "exitCode": 1,
+                            "finishedAt": timestamp,
+                            "reason": "Error",
+                            "startedAt": timestamp
+                        }
+                    },
+                    'restartCount': 0,
                 }
             ],
         }

From 295d79e1cf85fece72cee6d7073b3534f5e4e89c Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 23 May 2024 11:10:29 +0800
Subject: [PATCH 013/200] chore(pod): add restart count

---
 rootfs/api/models/app.py           | 17 ++++++++++-------
 rootfs/api/serializers/__init__.py |  2 ++
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index ed3974c1c..2c4b769b9 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -427,19 +427,22 @@ def list_pods(self, *args, **kwargs):
                     pods = []
             data = []
             for p in pods:
-                item = {}
                 labels = p['metadata']['labels']
-                item['name'] = p['metadata']['name']
-                item['state'] = str(self.scheduler().pod.state(p))
-                item['release'] = labels['version']
-                item['type'] = labels['type']
-                # set start time
                 if 'startTime' in p['status']:
                     started = p['status']['startTime']
                 else:
                     started = str(
                         datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT))
-                item['started'] = started
+                item = {
+                    'name': p['metadata']['name'], 'state': str(self.scheduler().pod.state(p)),
+                    'release': labels['version'], 'type': labels['type'], 'started': started,
+                    'ready': "%s/%s" % (
+                        len([1 for s in p["status"]["containerStatuses"] if s['ready']]),
+                        len(p["status"]["containerStatuses"]),
+                    ),
+                    'restarts': sum(
+                        [s['restartCount'] for s in p["status"]["containerStatuses"]]),
+                }
                 data.append(item)
             # sorting so latest start date is first
             data.sort(key=lambda x: x['started'], reverse=True)
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index aedfba500..c014b59fa 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -530,6 +530,8 @@ class PodSerializer(serializers.BaseSerializer):
     type = serializers.CharField()
     release = serializers.CharField(required=False)
     started = serializers.DateTimeField(required=False)
+    ready = serializers.CharField()
+    restarts = serializers.IntegerField()
 
     def to_representation(self, obj):
         return obj

From 5f5f32e102aded1cdebbd6d46ea3bbd67267e59b Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 23 May 2024 14:31:15 +0800
Subject: [PATCH 014/200] chore(pods): add describe pods command (#129)

---
 rootfs/api/models/app.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 2c4b769b9..54ab8fc7d 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -386,23 +386,23 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         return name
 
     def describe_pod(self, pod_name):
-        def get_commands_and_args(pod, container_name):
-            commands, args = [], []
+        def get_command_and_args(pod, container_name):
+            command, args = [], []
             for container in pod["spec"]["containers"]:
                 if container["name"] == container_name:
                     args = container.get("args", [])
-                    commands = container.get("commands", [])
+                    command = container.get("command", [])
                     break
-            return commands, args
+            return command, args
         result = []
         try:
             pod = self.scheduler().pod.get(self.id, pod_name).json()
             for status in pod["status"]["containerStatuses"]:
-                commands, args = get_commands_and_args(pod, status["name"])
+                command, args = get_command_and_args(pod, status["name"])
                 result.append({
                     "container": status["name"],
                     "image": status["image"],
-                    "commands": commands,
+                    "command": command,
                     "args": args,
                     "state": status["state"],
                     "lastState": status["lastState"],

From 2ade80608e1da7dc29f7a9c6ada956d76e0effdb Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 24 May 2024 15:55:33 +0800
Subject: [PATCH 015/200] chore(ps): restart types (#130)

---
 rootfs/api/models/app.py      | 15 ++++-----------
 rootfs/api/tests/test_pods.py | 12 +++++++-----
 rootfs/api/urls.py            |  3 ---
 rootfs/api/views.py           | 20 +++++++++++++++++---
 4 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 54ab8fc7d..ab706a74b 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -192,17 +192,10 @@ def restart(self, **kwargs):  # noqa
         """
         deployments = []
         app_settings = self.appsettings_set.latest()
-        if 'type' in kwargs and kwargs['type'] in self.structure:
-            if self.structure[kwargs['type']] > 0:
-                if kwargs['type'] in app_settings.canaries:
-                    deployments.append(self._get_job_id(kwargs['type'], True))
-                deployments.append(self._get_job_id(kwargs['type'], False))
-        else:
-            for scale_type, count in self.structure.items():
-                if count > 0:
-                    if scale_type in app_settings.canaries:
-                        deployments.append(self._get_job_id(kwargs['type'], True))
-                    deployments.append(self._get_job_id(scale_type, False))
+        if self.structure[kwargs['type']] > 0:
+            if kwargs['type'] in app_settings.canaries:
+                deployments.append(self._get_job_id(kwargs['type'], True))
+            deployments.append(self._get_job_id(kwargs['type'], False))
         try:
             tasks = [
                 functools.partial(
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index a846b6bdf..a5d53fb1f 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -773,13 +773,15 @@ def test_restart_pods(self, mock_requests):
         response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id))
         self.assertEqual(response.status_code, 204, response.data)
 
-        # restart only the workers
-        response = self.client.post('/v2/apps/{}/pods/worker/restart'.format(app_id))
+        # restart web and workers pods
+        body = {"types": "web,worker"}
+        response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id), body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        # restart only the web
-        response = self.client.post('/v2/apps/{}/pods/web/restart'.format(app_id))
-        self.assertEqual(response.status_code, 204, response.data)
+        # restart invalid ptypes
+        body = {"types": "web1"}
+        response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id), body)
+        self.assertEqual(response.status_code, 400, response.data)
 
         # restart only one of the web pods
         pods = application.list_pods(type='web')
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index c5b8ffc07..bbb2fe727 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -49,9 +49,6 @@
     re_path(
         r"^apps/(?P<id>{})/pods/restart/?$".format(settings.APP_URL_REGEX),
         views.PodViewSet.as_view({'post': 'restart'})),
-    re_path(
-        r"^apps/(?P<id>{})/pods/(?P<type>[-_\w.]+)/restart/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'post': 'restart'})),
     # list pods
     re_path(
         r"^apps/(?P<id>{})/pods/?$".format(settings.APP_URL_REGEX),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index f250d455b..8730e3f05 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -437,8 +437,22 @@ def list(self, *args, **kwargs):
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
 
-    def restart(self, *args, **kwargs):
-        restart_app.delay(self.get_app(), **kwargs)
+    def restart(self, request, *args, **kwargs):
+        app = self.get_app()
+        ptypes = []
+        types = request.data.get("types", "").split(",")
+        types = [ptype for ptype in set(types) if ptype != ""]
+        if not types:
+            # all ptypes need to restart
+            ptypes = app.structure.keys()
+        else:
+            ptypes = [ptype for ptype in types if ptype in app.structure]
+            invalid_ptypes = set(types) - set(ptypes)
+            if len(invalid_ptypes) != 0:
+                raise DryccException("process type {} is not included in procfile".
+                                     format(','.join(invalid_ptypes)))
+        for ptype in set(ptypes):
+            restart_app.delay(self.get_app(), **{"type": ptype})
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def describe(self, *args, **kwargs):
@@ -831,7 +845,7 @@ def path(self, request, *args, **kwargs):
         container_types = [_ for _ in path.keys()
                            if _ not in volume.app.procfile_types]
         if container_types:
-            raise DryccException("process type {} is not included in profile".
+            raise DryccException("process type {} is not included in procfile".
                                  format(','.join(container_types)))
         if set(path.items()).issubset(set(volume.path.items())):
             raise DryccException("mount path not changed")

From c8f8168921b81a28f6db4521504500f7921e9cb8 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 28 May 2024 10:01:20 +0800
Subject: [PATCH 016/200] fix(consumers): long log output error

---
 rootfs/api/consumers.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index 92d11ae89..48b57e73f 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -113,27 +113,24 @@ async def connect(self):
         self.running = False
         self.response = None
         self.conneted = True
+        self.buffer = b''
+        self.delimiter = b"\r\n"
         self.pod_id = self.scope["url_route"]["kwargs"]["pod_id"]
 
     def reader(self, sock):
+        self.buffer += sock.read()
         try:
-            delimiter, buffer = b"\r\n", sock.read()
-            if delimiter in buffer:
-                index = buffer.index(delimiter)
-                length = int(buffer[:index], base=16)
-                if len(buffer) - (index + len(delimiter)) < length:
-                    buffer += sock.read()
-            while buffer and self.conneted:
-                index = buffer.index(delimiter)
-                length = int(buffer[:index], base=16)
+            while self.buffer and self.buffer.endswith(self.delimiter):
+                index = self.buffer.index(self.delimiter)
+                length = int(self.buffer[:index], base=16)
                 if length == 0:
                     asyncio.create_task(self.close(code=1000))
                     break
-                start_pos = index + len(delimiter)
-                end_pos = start_pos + length + len(delimiter)
+                start_pos = index + len(self.delimiter)
+                end_pos = start_pos + length + len(self.delimiter)
                 asyncio.create_task(
-                    self.send(bytes_data=buffer[start_pos:end_pos].strip(delimiter)))
-                buffer = buffer[end_pos:]
+                    self.send(bytes_data=self.buffer[start_pos:end_pos].strip(self.delimiter)))
+                self.buffer = self.buffer[end_pos:]
         except BaseException:
             asyncio.create_task(self.close(code=1000))
 

From 5a448dfcb0c348a635dbb38413d003d910837b6a Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 29 May 2024 11:55:01 +0800
Subject: [PATCH 017/200] chore(volume): mount path allow '-' (#132)

---
 rootfs/api/serializers/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index c014b59fa..00b5b8fd7 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -55,7 +55,7 @@
     "Volume size limit format: <number><unit> or <number><unit>/<number><unit>, "
     "where unit = G, range: %sG~%sG"
 ) % (settings.KUBERNETES_LIMITS_MAX_VOLUME, settings.KUBERNETES_LIMITS_MIN_VOLUME)
-VOLUME_PATH_MATCH = re.compile(r'^\/(\w+\/?)+$', re.IGNORECASE)
+VOLUME_PATH_MATCH = re.compile(r'^\/([\w]+[\w-]*\/?)+$', re.IGNORECASE)
 METRIC_EVERY_MATCH = re.compile(r'^[1-9][0-9]*m$')
 HEALTHCHECK_MATCH = re.compile(r'^(livenessProbe|readinessProbe|startupProbe)$')
 HEALTHCHECK_MISMATCH_MSG = "Healthcheck pattern: %s" % HEALTHCHECK_MATCH.pattern

From a1251c14a85fa04d5af5b5f50a935e42ccd3f8b4 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 29 May 2024 18:47:35 +0800
Subject: [PATCH 018/200] fix(container): typed port error

---
 rootfs/api/models/app.py         | 14 +++----
 rootfs/api/models/release.py     | 47 ++--------------------
 rootfs/api/tests/test_release.py | 68 ++++++++++++--------------------
 3 files changed, 36 insertions(+), 93 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index ab706a74b..8783a59d0 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -23,7 +23,6 @@
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
 from api.utils import generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
-from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 from .gateway import Gateway, Route
 from .limit import LimitPlan
 from .config import Config
@@ -682,9 +681,9 @@ def _deploy(self, deploys, procfile_types, prev_release,
             err = '(app::deploy): {}'.format(e)
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
-        for procfile_type, value in deploys.items():
+        for procfile_type in deploys.keys():
             if procfile_type == PROCFILE_TYPE_WEB:  # http
-                target_port = int(value.get('envs', {}).get('PORT', DEFAULT_CONTAINER_PORT))
+                target_port = release.get_port(procfile_type)
                 self._create_default_ingress(target_port)
             service = self.service_set.filter(procfile_type=procfile_type).first()
             if not service:
@@ -953,14 +952,13 @@ def _build_env_vars(self, release, procfile_type):
         }
 
         default_env['SOURCE_VERSION'] = release.build.sha
-
-        # fetch application port and inject into ENV vars as needed
-        port = release.get_port()
-        if port:
-            default_env['PORT'] = port
         # merge envs on top of default to make envs win
         default_env.update(release.config.values)
         default_env.update(release.config.typed_values.get(procfile_type, {}))
+        # fetch application port and inject into ENV vars as needed
+        port = release.get_port(procfile_type)
+        if port:
+            default_env['PORT'] = port
         return default_env
 
     def _get_private_registry_config(self, image, registry=None):
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 879928012..ba4b8eee3 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -163,54 +163,15 @@ def new(self, user, config, build, summary=None, canary=False):
             build=build, version=new_version, summary=summary, canary=canary
         )
 
-    def get_port(self):
+    def get_port(self, procfile_type):
         """
         Get application port for a given release. If pulling from private registry
         then use default port or read from ENV var, otherwise attempt to pull from
         the container image
         """
-        try:
-            envs = self.config.values
-            creds = self.get_registry_auth()
-
-            if self.build.type == "buildpack":
-                self.log(
-                    'buildpack type detected. Defaulting to $PORT %s' % DEFAULT_CONTAINER_PORT)
-                return DEFAULT_CONTAINER_PORT
-
-            # application has registry auth - $PORT is required
-            if (creds is not None) or (settings.REGISTRY_LOCATION != 'on-cluster'):
-                if envs.get('PORT', None) is None:
-                    if not self.app.appsettings_set.latest().routable:
-                        return None
-                    raise DryccException(
-                        'PORT needs to be set in the application config '
-                        'when using a private registry'
-                    )
-
-                # User provided PORT
-                return int(envs.get('PORT'))
-
-            # If the user provides PORT
-            return int(envs.get('PORT', DEFAULT_CONTAINER_PORT))
-
-        except Exception as e:
-            raise DryccException(str(e)) from e
-
-    def get_registry_auth(self):
-        """
-        Gather login information for private registry if needed
-        """
-        auth = None
-        registry = self.config.registry
-        if registry.get('username', None):
-            auth = {
-                'username': registry.get('username', None),
-                'password': registry.get('password', None),
-                'email': self.owner.email
-            }
-
-        return auth
+        return int(self.config.typed_values.get(
+            procfile_type, {}).get(
+                'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
 
     def previous(self):
         """
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 0d5c838a6..958058a05 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -15,7 +15,6 @@
 from api.models.build import Build
 from api.models.release import Release
 from scheduler import KubeHTTPException
-from api.exceptions import DryccException
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
 
@@ -443,10 +442,7 @@ def test_release_get_port(self, mock_requests):
         release = app.release_set.latest()
 
         # when app is not routable, then it still return 5000
-        self.assertEqual(release.get_port(), 5000)
-
-        # when a buildpack type, default to 5000
-        self.assertEqual(release.get_port(), 5000)
+        self.assertEqual(release.get_port('web'), 5000)
 
         # switch to a dockerfile app or else it'll automatically default to 5000
         url = f'/v2/apps/{app_id}/builds'
@@ -459,11 +455,29 @@ def test_release_get_port(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
+        self.assertEqual(release.get_port('web'), 8080)
 
-        # check that the port number returned is an int, not a string
-        self.assertEqual(release.get_port(), 8080)
+        url = f'/v2/apps/{app_id}/config'
+        body = {'typed_values': json.dumps({"web": {'PORT': '9000'}})}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('web'), 9000)
 
-        # TODO(bacongobbler): test dockerfile ports
+        # not web procfile
+        self.assertEqual(release.get_port('task'), 8080)
+        url = f'/v2/apps/{app_id}/config'
+        body = {'values': json.dumps({'PORT': '9000'})}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('task'), 9000)
+        # set typed_values port
+        body = {'typed_values': json.dumps({"task": {'PORT': '9001'}})}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('task'), 9001)
 
     @override_settings(DRYCC_DEPLOY_HOOK_URLS=['http://drycc.rocks'])
     @mock.patch('api.models.release.logger')
@@ -575,45 +589,15 @@ def test_release_external_registry(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
-
-        self.assertEqual(release.get_port(), 3000)
-
+        self.assertEqual(release.get_port('web'), 3000)
         self.assertEqual(release.get_deploy_image(PROCFILE_TYPE_WEB), 'test/autotest/example')
 
-    @override_settings(REGISTRY_LOCATION="off-cluster")
-    def test_release_external_registry_no_port(self, mock_requests):
-        """
-        Test that an exception is raised when registry if off-cluster but
-        no port is provided.
-        """
-        app_id = self.create_app()
-
-        app = App.objects.get(id=app_id)
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'test/autotest/example', 'stack': 'container'}
+        url = f'/v2/apps/{app_id}/config'
+        body = {'typed_values': json.dumps({"web": {'PORT': '9000'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
-        self.assertEqual(data["results"][0]["state"], "crashed", data)
-
-        with self.assertRaises(
-            DryccException,
-            msg='PORT needs to be set in the config when using a private registry'
-        ):
-            release = app.release_set.latest()
-            release.get_port()
-
-        # Set routable to false and port should be None instead of error
-        response = self.client.post(
-            f'/v2/apps/{app.id}/settings',
-            {'routable': False}
-        )
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertFalse(app.appsettings_set.latest().routable)
-
         release = app.release_set.latest()
-        port = release.get_port()
-        self.assertIsNone(port)
+        self.assertEqual(release.get_port('web'), 9000)
 
     def test_diff_procfile_types(self, mock_requests):
         app_id = self.create_app()

From c90a08508af1c7346f2ab31a23ee6ac9eb17800b Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 30 May 2024 11:23:35 +0800
Subject: [PATCH 019/200] chore(prometheus): add container_spec_cpu_quota

---
 charts/controller/templates/_helpers.tpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index a06afb3d8..6c9330dd6 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -342,6 +342,7 @@ container_file_descriptors: [instance, namespace, pod, container]
 container_threads: [instance, namespace, pod, container]
 container_threads_max: [instance, namespace, pod, container]
 container_ulimits_soft: [instance, namespace, pod, container]
+container_spec_cpu_quota: [instance, namespace, pod, container]
 container_spec_cpu_period: [instance, namespace, pod, container]
 container_spec_cpu_shares: [instance, namespace, pod, container]
 container_spec_memory_limit_bytes: [instance, namespace, pod, container]

From 6e62226285bb2e36f18f092b85f3317a40276fef Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 30 May 2024 11:44:01 +0800
Subject: [PATCH 020/200] chore(prometheus): add monitoring items

---
 charts/controller/templates/_helpers.tpl | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 6c9330dd6..04ee1c86f 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -295,6 +295,7 @@ resources:
 
 {{/* Generate controller config default metrics */}}
 {{ define "controller.config.defaultMetrics" }}
+container_cpu_load_average_10s: [instance, namespace, pod, container]
 container_cpu_system_seconds_total: [instance, namespace, pod, container]
 container_cpu_usage_seconds_total: [instance, namespace, pod, container]
 container_cpu_user_seconds_total: [instance, namespace, pod, container]
@@ -336,6 +337,7 @@ container_network_transmit_bytes_total: [instance, namespace, pod, container]
 container_network_transmit_errors_total: [instance, namespace, pod, container]
 container_network_transmit_packets_dropped_total: [instance, namespace, pod, container]
 container_network_transmit_packets_total: [instance, namespace, pod, container]
+container_oom_events_total: [instance, namespace, pod, container]
 container_processes: [instance, namespace, pod, container]
 container_sockets: [instance, namespace, pod, container]
 container_file_descriptors: [instance, namespace, pod, container]
@@ -349,6 +351,7 @@ container_spec_memory_limit_bytes: [instance, namespace, pod, container]
 container_spec_memory_reservation_limit_bytes: [instance, namespace, pod, container]
 container_spec_memory_swap_limit_bytes: [instance, namespace, pod, container]
 container_start_time_seconds: [instance, namespace, pod, container]
+container_tasks_state: [instance, namespace, pod, container]
 container_last_seen: [instance, namespace, pod, container]
 container_accelerator_memory_used_bytes: [instance, namespace, pod, container]
 container_accelerator_memory_total_bytes: [instance, namespace, pod, container]

From 952079ad6acd08f62da1477553df6a871c42154a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 31 May 2024 11:03:20 +0800
Subject: [PATCH 021/200] chore(charts): add 64 vcpu

---
 charts/controller/templates/_helpers.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 04ee1c86f..5818dae74 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -256,7 +256,7 @@ resources:
 {{/* Generate controller config default limit plans */}}
 {{ define "controller.config.defaultLimitPlans" }}
 {{- $index := 0 }}
-{{- $cpus := tuple 1 2 4 8 16 32 }}
+{{- $cpus := tuple 1 2 4 8 16 32 64 }}
 {{- $scales := tuple 1 2 4 8 }}
 {{- range $cpu := $cpus }}
 {{- range $scale := $scales }}

From c15b425ebdb66baeb46abdf833e78dc3504e29fa Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 31 May 2024 14:50:12 +0800
Subject: [PATCH 022/200] fix(consumers): no data when follow is false

---
 rootfs/api/consumers.py | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index 48b57e73f..785f88217 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -9,6 +9,7 @@
 
 from asgiref.sync import sync_to_async
 
+from urllib3.response import HTTPResponse
 from kubernetes.client import Configuration, exceptions
 from kubernetes.client.api import core_v1_api
 from kubernetes.stream import stream
@@ -140,19 +141,26 @@ async def receive(self, text_data=None, bytes_data=None):
         self.running = True
         data = json.loads(text_data)
         args = (self.pod_id, self.id)
+        lines = data.get("lines", 100)
+        follow = data.get("follow", False)
         kwargs = {
-            "tail_lines": data.get("lines", 100),
-            "follow": data.get("follow", False),
+            "tail_lines": lines if lines < 1000 else 1000,
+            "follow": follow,
             "container": data.get("container", ""),
-            "_preload_content": False,
+            "_preload_content": not follow,
         }
-        loop = asyncio.get_event_loop()
         self.response = await sync_to_async(self.kubernetes.read_namespaced_pod_log)(
             *args, **kwargs)
-        loop.add_reader(self.response.connection.sock, self.reader, self.response.connection.sock)
+        if follow:
+            loop = asyncio.get_event_loop()
+            loop.add_reader(
+                self.response.connection.sock, self.reader, self.response.connection.sock)
+        else:
+            asyncio.create_task(self.send(text_data=self.response))
+            asyncio.create_task(self.close(code=1000))
 
     async def disconnect(self, close_code):
-        if self.response:
+        if isinstance(self.response, HTTPResponse):
             loop = asyncio.get_event_loop()
             loop.remove_reader(self.response.connection.sock)
             await sync_to_async(self.response.close)()

From f4a61174435d90fc254f031860fb2d30a0e72a53 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 5 Jun 2024 16:05:41 +0800
Subject: [PATCH 023/200] chore(release): append release summary

---
 rootfs/api/models/app.py          | 2 +-
 rootfs/api/models/build.py        | 2 +-
 rootfs/api/models/release.py      | 3 ++-
 rootfs/api/settings/production.py | 6 ++++++
 rootfs/api/urls.py                | 3 +++
 rootfs/api/views.py               | 5 ++++-
 6 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 8783a59d0..a68dfd6bd 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -255,7 +255,7 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
         except Exception as e:
             release.state = "crashed"
             release.failed = True
-            release.summary = "{} pipeline a release that failed".format(self.owner)
+            release.summary += "{} pipeline a release that failed".format(self.owner)
             release.exception = "error: {}".format(str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
         release.save()
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 3f7d0ba21..89b9bfc5e 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -90,7 +90,7 @@ def create_release(self, user, *args, **kwargs):
                 new_release = self.app.release_set.latest()
                 new_release.state = "crashed"
                 new_release.failed = True
-                new_release.summary = "{} deployed {} which failed".format(
+                new_release.summary += "{} deployed {} which failed".format(
                     self.owner, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index ba4b8eee3..d04e36150 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -223,7 +223,8 @@ def rollback(self, user, version=None):
                 new_release = self.app.release_set.latest()
                 new_release.state = "crashed"
                 new_release.failed = True
-                new_release.summary = "{} performed roll back to a release that failed".format(self.owner)  # noqa
+                new_release.summary += "{} performed roll back to a release that failed".format(
+                    self.owner)
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
                 new_release.save()
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 22910641e..d075d33e2 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -301,6 +301,12 @@
 
 IMAGE_PULL_POLICY = os.environ.get('IMAGE_PULL_POLICY', "IfNotPresent")
 
+# Drycc filer image
+# Provide get and put operations for `drycc volumes:client`
+DRYCC_FILER_IMAGE = os.environ.get('DRYCC_FILER_IMAGE', 'registry.drycc.cc/drycc/filer:canary')
+DRYCC_FILER_DURATION = int(os.environ.get('DRYCC_FILER_DURATION', '3600'))
+DRYCC_FILER_WAITTIME = int(os.environ.get('DRYCC_FILER_WAITTIME', '1200'))
+
 # True, true, yes, y and more evaluate to True
 # False, false, no, n and more evaluate to False
 # see the above for all available options
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index bbb2fe727..5920d4f99 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -110,6 +110,9 @@
     re_path(
         r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/path/?$".format(settings.APP_URL_REGEX),
         views.AppVolumesViewSet.as_view({'patch': 'path'})),
+    re_path(
+        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/client/?$".format(settings.APP_URL_REGEX),
+        views.AppVolumesViewSet.as_view({'get': 'client', 'post': 'client'})),
     # application resources
     re_path(r"^resources/services/?$", views.AppResourcesViewSet.as_view({'get': 'services'})),
     re_path(
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 8730e3f05..1a0d19ee9 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -415,7 +415,7 @@ def post_save(self, config):
             if 'release' in locals():
                 release.state = "crashed"
                 release.failed = True
-                release.summary = "{} deployed a config that failed".format(self.request.user)
+                release.summary += "{} deployed a config that failed".format(self.request.user)
                 # Get the exception that has occured
                 release.exception = "error: {}".format(str(e))
                 release.save()
@@ -822,6 +822,9 @@ def get_object(self):
                                  app__id=self.kwargs['id'],
                                  name=self.kwargs['name'])
 
+    def client(self, request, **kwargs):
+        pass
+
     def expand(self, request, **kwargs):
         volume = self.get_object()
         volume.expand(request.data['size'])

From 7d5a61e4476ebb4fbffe52ebe602c0ef18b4d2a0 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 11 Jun 2024 10:14:31 +0800
Subject: [PATCH 024/200] chore(pod): add CONTAINER_IP env

---
 rootfs/scheduler/resources/pod.py | 42 +++++++++++++++++++------------
 1 file changed, 26 insertions(+), 16 deletions(-)

diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 89213cb32..5363c885b 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -221,34 +221,44 @@ def _set_container(self, namespace, container_name, data, **kwargs):
             for key in env.keys():
                 item = {
                     "name": key,
-                    "valueFrom": {
-                        "secretKeyRef": {
-                            "name": secret_name,
-                            # k8s doesn't allow _ so translate to -, see above
-                            "key": key.lower().replace('_', '-')
-                        }
-                    }
+                    "valueFrom": {"secretKeyRef": {
+                        "name": secret_name,
+                        # k8s doesn't allow _ so translate to -, see above
+                        "key": key.lower().replace('_', '-')
+                    }}
                 }
-
                 # add value to env hash. Overwrite hardcoded values if need be
                 match = next((k for k, e in enumerate(data["env"]) if e['name'] == key), None)
                 if match is not None:
                     data["env"][match] = item
                 else:
                     data["env"].append(item)
-
-        # Inject debugging if workflow is in debug mode
-        if os.environ.get("DRYCC_DEBUG", False):
-            data["env"].append({
-                "name": "DRYCC_DEBUG",
-                "value": "1"
-            })
-
+        # set container default env
+        self._set_container_default_env(data)
         # list sorted by dict key name
         data['env'].sort(key=operator.itemgetter('name'))
         self._set_health_checks(data, env, **kwargs)
         self._set_lifecycle_hooks(data, env, **kwargs)
 
+    def _set_container_default_env(self, data):
+        # set fields env
+        fields = {"CONTAINER_IP": "status.podIP"}
+        for key, value in fields.items():
+            item = {
+                "name": key,
+                "valueFrom": {"fieldRef": {"fieldPath": value}}
+            }
+            # add value to env hash. Overwrite hardcoded values if need be
+            match = next((k for k, e in enumerate(data["env"]) if e['name'] == key), None)
+            if match is not None:
+                data["env"][match] = item
+            else:
+                data["env"].append(item)
+        # Inject debugging if workflow is in debug mode
+        if os.environ.get("DRYCC_DEBUG", False):
+            data["env"].append({"name": "DRYCC_DEBUG", "value": "1"})
+        return data
+
     @staticmethod
     def _get_termination_grace_period(kwargs):
         """ return termination grace period """

From bf0da33a4de38ad50e9f481613a9f73a253da37d Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 19 Jun 2024 17:53:10 +0800
Subject: [PATCH 025/200] fix(serializer): list type representation error

---
 rootfs/api/serializers/__init__.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 00b5b8fd7..e580ad355 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -86,19 +86,22 @@ def to_internal_value(self, data):
 
     def to_representation(self, obj):
         """Serialize the field's JSON data, for read operations."""
-        for k, v in obj.items():
-            if v is None:  # NoneType is used to unset a value
-                continue
-
-            try:
-                if isinstance(v, (dict, list)):
-                    self.to_representation(v)
-                elif self.convert_to_str:
-                    obj[k] = str(v)
-            except ValueError:
-                obj[k] = v
-                # Do nothing, the validator will catch this later
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if v is None:  # NoneType is used to unset a value
+                    continue
 
+                try:
+                    if isinstance(v, (dict, list)):
+                        self.to_representation(v)
+                    elif self.convert_to_str:
+                        obj[k] = str(v)
+                except ValueError:
+                    obj[k] = v
+                    # Do nothing, the validator will catch this later
+        elif isinstance(obj, list):
+            for i in range(len(obj)):
+                obj[i] = self.to_representation(obj[i])
         return obj
 
 

From 47a943515b033b0742a1ae86011cf619cd151df0 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 20 Jun 2024 10:39:44 +0800
Subject: [PATCH 026/200] chore(ps:logs): support pod all logs

---
 rootfs/api/consumers.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index 785f88217..e1552043d 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -141,14 +141,15 @@ async def receive(self, text_data=None, bytes_data=None):
         self.running = True
         data = json.loads(text_data)
         args = (self.pod_id, self.id)
-        lines = data.get("lines", 100)
+        lines = data.get("lines", 300)
         follow = data.get("follow", False)
         kwargs = {
-            "tail_lines": lines if lines < 1000 else 1000,
             "follow": follow,
             "container": data.get("container", ""),
             "_preload_content": not follow,
         }
+        if lines > 0:
+            kwargs["tail_lines"] = lines
         self.response = await sync_to_async(self.kubernetes.read_namespaced_pod_log)(
             *args, **kwargs)
         if follow:

From 2047d4173041c02f0ca3ff264438748cf9b907b3 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 25 Jun 2024 10:04:58 +0800
Subject: [PATCH 027/200] chore(release): pre_release is incorrect

---
 rootfs/api/models/app.py         |  2 ++
 rootfs/api/models/build.py       |  2 ++
 rootfs/api/models/release.py     |  4 +++-
 rootfs/api/tests/test_release.py | 24 ++++++++++++++++--------
 rootfs/api/views.py              |  2 ++
 5 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index a68dfd6bd..0aa664f02 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -255,6 +255,8 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
         except Exception as e:
             release.state = "crashed"
             release.failed = True
+            if release.summary:
+                release.summary += " "
             release.summary += "{} pipeline a release that failed".format(self.owner)
             release.exception = "error: {}".format(str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 89b9bfc5e..32df3a48b 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -90,6 +90,8 @@ def create_release(self, user, *args, **kwargs):
                 new_release = self.app.release_set.latest()
                 new_release.state = "crashed"
                 new_release.failed = True
+                if new_release.summary:
+                    new_release.summary += " "
                 new_release.summary += "{} deployed {} which failed".format(
                     self.owner, str(self.uuid)[:7])
                 # Get the exception that has occured
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index d04e36150..fedf74eb3 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -185,7 +185,7 @@ def previous(self):
 
         try:
             # Get the Release previous to this one
-            prev_release = releases.filter(failed=False).latest()
+            prev_release = releases.filter(failed=False, state="succeed").latest()
         except Release.DoesNotExist:
             prev_release = None
         return prev_release
@@ -223,6 +223,8 @@ def rollback(self, user, version=None):
                 new_release = self.app.release_set.latest()
                 new_release.state = "crashed"
                 new_release.failed = True
+                if new_release.summary:
+                    new_release.summary += " "
                 new_release.summary += "{} performed roll back to a release that failed".format(
                     self.owner)
                 # Get the exception that has occured
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 958058a05..009ebf39b 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -638,7 +638,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertIsNone(release.diff_procfile_types())
         # test has image
@@ -669,7 +670,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), {'web', 'worker'})
         # test has image
@@ -700,7 +702,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), {'web'})
         # test has image
@@ -731,7 +734,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), {'worker'})
         # test no image
@@ -771,7 +775,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), {'web', 'worker', 'worker-sync'})
         # test has image
@@ -805,7 +810,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), {'worker', 'task', 'worker-sync', 'web'})
         # test has image
@@ -839,7 +845,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), {'task'})
         # test has image
@@ -873,7 +880,8 @@ def test_diff_procfile_types(self, mock_requests):
             owner=user,
             app=app,
             config=app.config_set.latest(),
-            build=build
+            build=build,
+            state="succeed"
         )
         self.assertEqual(release.diff_procfile_types(), set())
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 1a0d19ee9..efcc9f4f1 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -415,6 +415,8 @@ def post_save(self, config):
             if 'release' in locals():
                 release.state = "crashed"
                 release.failed = True
+                if release.summary:
+                    release.summary += " "
                 release.summary += "{} deployed a config that failed".format(self.request.user)
                 # Get the exception that has occured
                 release.exception = "error: {}".format(str(e))

From d0f6cc3d76b322d280501de57a63712edd192d09 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 24 Jun 2024 15:36:27 +0800
Subject: [PATCH 028/200] feat(volumes): add volumes client support

---
 rootfs/api/urls.py  |  3 +-
 rootfs/api/utils.py | 76 +++++++++++++++++++++++++++++++++++++++++++++
 rootfs/api/views.py | 18 +++++++++--
 3 files changed, 94 insertions(+), 3 deletions(-)

diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 5920d4f99..893379c49 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -111,7 +111,8 @@
         r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/path/?$".format(settings.APP_URL_REGEX),
         views.AppVolumesViewSet.as_view({'patch': 'path'})),
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/client/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/files(/?(?P<path>[\S]+)/?)?$".format(
+            settings.APP_URL_REGEX),
         views.AppVolumesViewSet.as_view({'get': 'client', 'post': 'client'})),
     # application resources
     re_path(r"^resources/services/?$", views.AppResourcesViewSet.as_view({'get': 'services'})),
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index de4c63ff7..be649c16a 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -16,9 +16,13 @@
 import jsonschema
 from copy import deepcopy
 from django.db import models
+from django.conf import settings
+from django.core.cache import cache
+from requests.auth import HTTPBasicAuth
 from requests_toolbelt import user_agent
 from api import __version__ as drycc_version
 from rest_framework.exceptions import ValidationError
+from scheduler import KubeException
 
 logger = logging.getLogger(__name__)
 
@@ -196,6 +200,78 @@ def validate_json(value, schema, raise_exception=ValidationError):
     return value
 
 
+class VolumeClient(object):
+
+    def __init__(self, app_id, volume, scheduler):
+        self.bind = ":9000"
+        self.path = "/data"
+        self.app_id = app_id
+        self.volume = volume
+        self.scheduler = scheduler
+
+    @property
+    def server(self):
+        _server = cache.get(self.cache_key, None)
+        if not _server or not self.health(_server):
+            self.cleanup()
+            k8s_volume = {"name": self.volume.name}
+            if self.volume.type == "csi":
+                k8s_volume.update({"persistentVolumeClaim": {"claimName": self.volume.name}})
+            else:
+                k8s_volume.update(self.volume.parameters)
+            username, password = random_string(32), random_string(32)
+            self.scheduler.pod.create(self.app_id, self.pod_name, settings.DRYCC_FILER_IMAGE, **{
+                "args": [
+                    "filer",
+                    "--bind", self.bind, "--path", self.path,
+                    "--duration", f"{settings.DRYCC_FILER_DURATION}",
+                    "--waittime", f"{settings.DRYCC_FILER_WAITTIME}",
+                    "--username", f"{username}", "--password", f"{password}",
+                ],
+                "app_type": "filer",
+                "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
+                "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
+                "image_pull_policy": "Always",
+            })
+            address = self.scheduler.pod.get(self.app_id, self.pod_name).json()["status"]["podIP"]
+            _server = {"address": address, "username": username, "password": password}
+            cache.set(self.cache_key, _server, timeout=settings.DRYCC_FILER_DURATION)
+        return _server
+
+    @property
+    def pod_name(self):
+        return f"drycc-filer-{self.volume.name}"
+
+    @property
+    def cache_key(self):
+        return f"filer:{self.pod_name}"
+
+    def health(self, server):
+        try:
+            return self.request(
+                "OPTIONS", server, timeout=2).headers.get('server') == 'drycc-filer'
+        except requests.exceptions.Timeout:
+            return False
+
+    def cleanup(self):
+        try:
+            self.scheduler.pod.delete(self.app_id, self.pod_name)
+        except KubeException:
+            pass
+
+    def request(self, method, server, path="/", **kwargs):
+        cache.touch(self.cache_key, timeout=settings.DRYCC_FILER_DURATION)
+        url = f"http://{server["address"]}:{self.bind.split(":")[1]}/{path}"
+        kwargs["auth"] = HTTPBasicAuth(server["username"], server["password"])
+        return get_session().request(method, url, **kwargs)
+
+    def get(self, path, **kwargs):
+        return self.request("GET", self.server, path, **kwargs)
+
+    def post(self, path, **kwargs):
+        return self.request("POST", self.server, path, **kwargs)
+
+
 if __name__ == "__main__":
     import doctest
     doctest.testmod()
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index efcc9f4f1..97b8dc368 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -23,6 +23,8 @@
 from rest_framework.permissions import IsAuthenticated, AllowAny
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
+from rest_framework.decorators import parser_classes
+from rest_framework.parsers import MultiPartParser
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner
@@ -35,7 +37,7 @@
 from social_django.utils import psa
 from social_django.views import _do_login
 from social_core.utils import setting_name
-from api import admissions
+from api import admissions, utils
 from api.backend import OauthCacheManager
 from api.apps_extra.social_core.actions import do_auth, do_complete
 
@@ -824,8 +826,20 @@ def get_object(self):
                                  app__id=self.kwargs['id'],
                                  name=self.kwargs['name'])
 
+    @method_decorator(parser_classes([MultiPartParser]))
     def client(self, request, **kwargs):
-        pass
+        path = self.kwargs.get('path', '')
+        volume = self.get_object()
+        client = utils.VolumeClient(volume.app.id, volume, volume.app.scheduler())
+        if request.method == "GET":
+            response = client.get(path, stream=True)
+            return StreamingHttpResponse(
+                status=response.status_code, streaming_content=response.iter_content(),
+                content_type=response.headers.get('Content-Type', 'application/octet-stream'))
+        elif request.method == "POST":
+            client.post(path, files=request.FILES)
+            return HttpResponse(status=status.HTTP_204_NO_CONTENT)
+        return HttpResponse(status=status.HTTP_405_METHOD_NOT_ALLOWED)
 
     def expand(self, request, **kwargs):
         volume = self.get_object()

From cd9b997225802b10e0ab6fab8b8ea7a244d79f65 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 28 Jun 2024 15:32:28 +0800
Subject: [PATCH 029/200] chore(controller): pipeline and httproute
 optimization

* fix(config): executing pipeline should not save db

* chore(resource): serviceinstance status message is null when start

* chore(gateway): httproute support timeout config
---
 rootfs/api/models/resource.py           | 5 +++--
 rootfs/api/serializers/schemas/rules.py | 4 ++++
 rootfs/api/tests/rules/http_rules.json  | 6 +++++-
 rootfs/api/views.py                     | 1 +
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index 2ffbdb32e..972b2ff56 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -192,8 +192,9 @@ def message(self):
                 self.app.id, self.name)
             if resp.status_code != 200:
                 message = ""
-            message = resp.json().get("status", {}).get("conditions", [{}])[-1].\
-                get("message", "")
+            conditions = resp.json().get("status", {}).get("conditions")
+            message = conditions[-1].get("message", "") \
+                if conditions and isinstance(conditions[-1], dict) else ""
             return message
         except KubeException as e:
             logger.info("retrieve instance info error: {}".format(e))
diff --git a/rootfs/api/serializers/schemas/rules.py b/rootfs/api/serializers/schemas/rules.py
index 97f9f12e0..b9b10f254 100644
--- a/rootfs/api/serializers/schemas/rules.py
+++ b/rootfs/api/serializers/schemas/rules.py
@@ -193,6 +193,10 @@
                     "required": ["name"],
                     "additionalProperties": False
                 }
+            },
+            "timeouts": {
+                "request": {"type": "string"},
+                "backendRequest": {"type": "string"}
             }
         },
         "required": ["backendRefs"],
diff --git a/rootfs/api/tests/rules/http_rules.json b/rootfs/api/tests/rules/http_rules.json
index 4021ea883..5c4371350 100644
--- a/rootfs/api/tests/rules/http_rules.json
+++ b/rootfs/api/tests/rules/http_rules.json
@@ -27,7 +27,11 @@
 		"backendRefs": [{
 			"name": "echo",
 			"port": 8080
-		}]
+		}],
+		"timeouts": {
+			"backendRequest": "60s",
+			"request": "60s"
+		}
 	}],
 	"canary": [{
 		"matches": [{
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 97b8dc368..84fc7eae9 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -397,6 +397,7 @@ class ConfigViewSet(ReleasableViewSet):
     def post_save(self, config):
         latest_release = config.app.release_set.filter(failed=False).latest()
         if latest_release.build is not None and latest_release.state == "created":
+            config.delete()
             raise DryccException('There is an executing pipeline, please wait')
         try:
             release = latest_release.new(

From 6ad0ca9bb81660f37b5cfaef7d7e8d9560a44e77 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 1 Jul 2024 14:00:41 +0800
Subject: [PATCH 030/200] chore(volumes): add add filer delete

---
 rootfs/api/filer.py               |  81 +++++++++++++++++++++++
 rootfs/api/monitor.py             |  39 ++++++-----
 rootfs/api/urls.py                |   9 ++-
 rootfs/api/utils.py               | 106 +++++++++++-------------------
 rootfs/api/views.py               |  64 ++++++++++++------
 rootfs/scheduler/resources/pod.py |   9 +--
 6 files changed, 192 insertions(+), 116 deletions(-)
 create mode 100644 rootfs/api/filer.py

diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
new file mode 100644
index 000000000..211b69a73
--- /dev/null
+++ b/rootfs/api/filer.py
@@ -0,0 +1,81 @@
+import uuid
+import requests
+from django.conf import settings
+from django.core.cache import cache
+from requests.auth import HTTPBasicAuth
+
+from .utils import random_string, get_session, CacheLock
+
+
+class FilerClient(object):
+
+    def __init__(self, app_id, volume, scheduler):
+        self.bind = ":9000"
+        self.path = "/data"
+        self.app_id = app_id
+        self.volume = volume
+        self.scheduler = scheduler
+
+    @property
+    def server(self):
+        lock_key = f"filer:lock:{self.app_id}:{self.volume.name}"
+        lock = CacheLock(lock_key)
+        try:
+            lock.acquire()
+            _server = cache.get(self.cache_key, None)
+            if not _server or not self.health(_server):
+                _server = self.get_server()
+                cache.set(self.cache_key, _server, timeout=settings.DRYCC_FILER_DURATION)
+        finally:
+            lock.release()
+        return _server
+
+    @property
+    def cache_key(self):
+        return f"filer:{self.app_id}:{self.volume.name}"
+
+    def get_server(self):
+        pod_name = f"drycc-filer-{uuid.uuid4().hex}"
+        k8s_volume = {"name": self.volume.name}
+        if self.volume.type == "csi":
+            k8s_volume.update({"persistentVolumeClaim": {"claimName": self.volume.name}})
+        else:
+            k8s_volume.update(self.volume.parameters)
+        username, password = random_string(32), random_string(32)
+        self.scheduler.pod.create(self.app_id, pod_name, settings.DRYCC_FILER_IMAGE, **{
+            "args": [
+                "filer",
+                "--bind", self.bind, "--path", self.path,
+                "--duration", f"{settings.DRYCC_FILER_DURATION}",
+                "--waittime", f"{settings.DRYCC_FILER_WAITTIME}",
+                "--username", f"{username}", "--password", f"{password}",
+            ],
+            "labels": {"app": self.app_id, "pod": pod_name, "volume": self.volume.name},
+            "app_type": "filer", "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
+            "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
+            "restart_policy": "Never", "image_pull_policy": "Always",
+        })
+        address = self.scheduler.pod.get(self.app_id, pod_name).json()["status"]["podIP"]
+        return {"address": address, "username": username, "password": password}
+
+    def health(self, server):
+        try:
+            return self.request(
+                "OPTIONS", server, timeout=2).headers.get('server') == 'drycc-filer'
+        except requests.exceptions.Timeout:
+            return False
+
+    def request(self, method, server, path="/", **kwargs):
+        cache.touch(self.cache_key, timeout=settings.DRYCC_FILER_DURATION)
+        url = f"http://{server["address"]}:{self.bind.split(":")[1]}/{path}"
+        kwargs["auth"] = HTTPBasicAuth(server["username"], server["password"])
+        return get_session().request(method, url, **kwargs)
+
+    def get(self, path, **kwargs):
+        return self.request("GET", self.server, path, **kwargs)
+
+    def post(self, path, **kwargs):
+        return self.request("POST", self.server, path, **kwargs)
+
+    def delete(self, path, **kwargs):
+        return self.request("DELETE", self.server, path, **kwargs)
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 7b507ade6..e56431095 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -1,4 +1,5 @@
 import time
+import aiohttp
 import requests
 from typing import Iterator, Dict
 from contextlib import closing
@@ -114,29 +115,27 @@
 """
 
 
-def last_metrics(namespace):
+async def last_metrics(namespace):
     if not settings.DRYCC_METRICS_CONFIG:
-        return ()
+        return
     promql = query_last_metrics_promql_tpl % (
-        '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
-        namespace,
-        '1m'
-    )
+        '|'.join(settings.DRYCC_METRICS_CONFIG.keys()), namespace, '1m')
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
     params = {"query": promql, "start": int(time.time() - 60)}
-    response = requests.get(
-        urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query"),
-        params=params
-    )
-    if response.status_code != 200:
-        return ()
-    return ('%s{%s} %s\n' % (
-        item['metric']['__name__'],
-        ','.join([
-            f'{key}="{value}"' for key, value in item['metric'].items()
-            if key in settings.DRYCC_METRICS_CONFIG[item['metric']['__name__']]
-          ]),
-        item['value'][1],
-    ) for item in response.json()['data']['result'])
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, params=params) as response:
+            if response.status != 200:
+                return
+            items = await response.json()
+            for item in items['data']['result']:
+                yield '%s{%s} %s\n' % (
+                    item['metric']['__name__'],
+                    ','.join([
+                        f'{key}="{value}"' for key, value in item['metric'].items()
+                        if key in settings.DRYCC_METRICS_CONFIG[item['metric']['__name__']]
+                    ]),
+                    item['value'][1]
+                )
 
 
 def query_loadbalancer(namespaces: Iterator[str],
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 893379c49..ed84169a5 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -110,10 +110,15 @@
     re_path(
         r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/path/?$".format(settings.APP_URL_REGEX),
         views.AppVolumesViewSet.as_view({'patch': 'path'})),
+    # application filer
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/files(/?(?P<path>[\S]+)/?)?$".format(
+        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/client/?$".format(
             settings.APP_URL_REGEX),
-        views.AppVolumesViewSet.as_view({'get': 'client', 'post': 'client'})),
+        views.AppFilerClientViewSet.as_view({'get': 'list', 'post': 'create'})),
+    re_path(
+        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/client/(?P<path>[\S]+)$".format(
+            settings.APP_URL_REGEX),
+        views.AppFilerClientViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
     # application resources
     re_path(r"^resources/services/?$", views.AppResourcesViewSet.as_view({'get': 'services'})),
     re_path(
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index be649c16a..e732b8677 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -3,6 +3,8 @@
 """
 import os
 import re
+import uuid
+import time
 import base64
 import string
 import concurrent
@@ -16,13 +18,12 @@
 import jsonschema
 from copy import deepcopy
 from django.db import models
-from django.conf import settings
 from django.core.cache import cache
-from requests.auth import HTTPBasicAuth
+from asgiref.sync import sync_to_async
 from requests_toolbelt import user_agent
 from api import __version__ as drycc_version
 from rest_framework.exceptions import ValidationError
-from scheduler import KubeException
+
 
 logger = logging.getLogger(__name__)
 
@@ -200,76 +201,47 @@ def validate_json(value, schema, raise_exception=ValidationError):
     return value
 
 
-class VolumeClient(object):
+class CacheLock(object):
 
-    def __init__(self, app_id, volume, scheduler):
-        self.bind = ":9000"
-        self.path = "/data"
-        self.app_id = app_id
-        self.volume = volume
-        self.scheduler = scheduler
+    def __init__(self, key):
+        self.key = key
+        self.value = uuid.uuid4().hex
 
-    @property
-    def server(self):
-        _server = cache.get(self.cache_key, None)
-        if not _server or not self.health(_server):
-            self.cleanup()
-            k8s_volume = {"name": self.volume.name}
-            if self.volume.type == "csi":
-                k8s_volume.update({"persistentVolumeClaim": {"claimName": self.volume.name}})
-            else:
-                k8s_volume.update(self.volume.parameters)
-            username, password = random_string(32), random_string(32)
-            self.scheduler.pod.create(self.app_id, self.pod_name, settings.DRYCC_FILER_IMAGE, **{
-                "args": [
-                    "filer",
-                    "--bind", self.bind, "--path", self.path,
-                    "--duration", f"{settings.DRYCC_FILER_DURATION}",
-                    "--waittime", f"{settings.DRYCC_FILER_WAITTIME}",
-                    "--username", f"{username}", "--password", f"{password}",
-                ],
-                "app_type": "filer",
-                "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
-                "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
-                "image_pull_policy": "Always",
-            })
-            address = self.scheduler.pod.get(self.app_id, self.pod_name).json()["status"]["podIP"]
-            _server = {"address": address, "username": username, "password": password}
-            cache.set(self.cache_key, _server, timeout=settings.DRYCC_FILER_DURATION)
-        return _server
-
-    @property
-    def pod_name(self):
-        return f"drycc-filer-{self.volume.name}"
-
-    @property
-    def cache_key(self):
-        return f"filer:{self.pod_name}"
-
-    def health(self, server):
-        try:
-            return self.request(
-                "OPTIONS", server, timeout=2).headers.get('server') == 'drycc-filer'
-        except requests.exceptions.Timeout:
-            return False
+    def acquire(self, blocking=True, timeout=120):
+        value = None
+        for _ in range(timeout):
+            value = cache.get_or_set(self.key, self.value, timeout)
+            if blocking or value == self.value:
+                break
+            time.sleep(1)
+        return value == self.value
 
-    def cleanup(self):
-        try:
-            self.scheduler.pod.delete(self.app_id, self.pod_name)
-        except KubeException:
-            pass
+    def release(self):
+        value = cache.get(self.key, None)
+        if value == self.value:
+            return
+        cache.delete(self.key)
+
+
+class SyncIterToAsyncIter(object):
+
+    def __init__(self, iter_content):
+        self.iter_content = iter_content
+
+    def __aiter__(self):
+        return self
 
-    def request(self, method, server, path="/", **kwargs):
-        cache.touch(self.cache_key, timeout=settings.DRYCC_FILER_DURATION)
-        url = f"http://{server["address"]}:{self.bind.split(":")[1]}/{path}"
-        kwargs["auth"] = HTTPBasicAuth(server["username"], server["password"])
-        return get_session().request(method, url, **kwargs)
+    async def __anext__(self):
+        @sync_to_async
+        def async_next():
+            try:
+                return next(self.iter_content)
+            except StopIteration:
+                raise StopAsyncIteration
+        return await async_next()
 
-    def get(self, path, **kwargs):
-        return self.request("GET", self.server, path, **kwargs)
 
-    def post(self, path, **kwargs):
-        return self.request("POST", self.server, path, **kwargs)
+iter_to_aiter = SyncIterToAsyncIter
 
 
 if __name__ == "__main__":
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 84fc7eae9..da1f93bb2 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -23,7 +23,6 @@
 from rest_framework.permissions import IsAuthenticated, AllowAny
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
-from rest_framework.decorators import parser_classes
 from rest_framework.parsers import MultiPartParser
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
@@ -33,11 +32,11 @@
 from django.views.decorators.cache import never_cache
 from django.contrib.auth import REDIRECT_FIELD_NAME
 from django.views.decorators.csrf import csrf_exempt
-from django.http.response import StreamingHttpResponse
+from django.http.response import FileResponse, StreamingHttpResponse
 from social_django.utils import psa
 from social_django.views import _do_login
 from social_core.utils import setting_name
-from api import admissions, utils
+from api import admissions, utils, filer
 from api.backend import OauthCacheManager
 from api.apps_extra.social_core.actions import do_auth, do_complete
 
@@ -827,21 +826,6 @@ def get_object(self):
                                  app__id=self.kwargs['id'],
                                  name=self.kwargs['name'])
 
-    @method_decorator(parser_classes([MultiPartParser]))
-    def client(self, request, **kwargs):
-        path = self.kwargs.get('path', '')
-        volume = self.get_object()
-        client = utils.VolumeClient(volume.app.id, volume, volume.app.scheduler())
-        if request.method == "GET":
-            response = client.get(path, stream=True)
-            return StreamingHttpResponse(
-                status=response.status_code, streaming_content=response.iter_content(),
-                content_type=response.headers.get('Content-Type', 'application/octet-stream'))
-        elif request.method == "POST":
-            client.post(path, files=request.FILES)
-            return HttpResponse(status=status.HTTP_204_NO_CONTENT)
-        return HttpResponse(status=status.HTTP_405_METHOD_NOT_ALLOWED)
-
     def expand(self, request, **kwargs):
         volume = self.get_object()
         volume.expand(request.data['size'])
@@ -877,6 +861,46 @@ def path(self, request, *args, **kwargs):
         return Response(serializer.data)
 
 
+class AppFilerClientViewSet(BaseDryccViewSet):
+    """RESTful views for volumes apps with collaborators."""
+    model = models.volume.Volume
+    parser_classes = [MultiPartParser]
+
+    def get_client(self):
+        volume = get_object_or_404(
+            models.volume.Volume, app__id=self.kwargs['id'], name=self.kwargs['name'])
+        return filer.FilerClient(volume.app.id, volume, volume.app.scheduler())
+
+    def list(self, request, **kwargs):
+        path = request.query_params.get('path', '')
+        client = self.get_client()
+        results = client.get(path, params={"action": "list"}).json()
+        # fake out pagination for now
+        pagination = {'results': results, 'count': len(results)}
+        return Response(data=pagination)
+
+    def retrieve(self, request, **kwargs):
+        path = self.kwargs.get('path', '')
+        client = self.get_client()
+        response = client.get(path, stream=True, params={"action": "get"})
+        return FileResponse(
+            status=response.status_code,
+            streaming_content=utils.iter_to_aiter(response.iter_content()),
+        )
+
+    def create(self, request, **kwargs):
+        path = request.data.get('path', '')
+        client = self.get_client()
+        response = client.post(path, files=request.FILES)
+        return Response(data=response.content, status=response.status_code)
+
+    def destroy(self, request, **kwargs):
+        path = self.kwargs.get('path', '')
+        client = self.get_client()
+        response = client.delete(path)
+        return Response(data=response.content, status=response.status_code)
+
+
 class AppResourcesViewSet(AppResourceViewSet):
     """RESTful views for resources apps with collaborators."""
     model = models.resource.Resource
@@ -1193,8 +1217,6 @@ def status(self, request, **kwargs):
     @method_decorator(vary_on_headers("Authorization"))
     def metric(self, request, **kwargs):
         app_id = self._get_app().id
-        streaming_content = monitor.last_metrics(app_id)
         return StreamingHttpResponse(
-            content_type='application/json',
-            streaming_content=streaming_content
+            streaming_content=monitor.last_metrics(app_id)
         )
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 5363c885b..3b68a88db 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -134,12 +134,9 @@ def manifest(self, namespace, name, image, **kwargs):
         app_type = kwargs.get('app_type')
 
         # labels that represent the pod(s)
-        labels = {
-            'app': namespace,
-            'version': kwargs.get('version'),
-            'type': app_type,
-            'heritage': 'drycc',
-        }
+        default_labels = {'app': namespace, 'version': kwargs.get('version'), 'heritage': 'drycc'}
+        labels = kwargs.get('labels', default_labels)
+        labels.update({'type': app_type})
 
         # create base pod structure
         manifest = {

From a67bda1ddc7631022837c2ec0addc5d4c3363a30 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 1 Jul 2024 14:01:02 +0800
Subject: [PATCH 031/200] chore(deps): bump djangorestframework from 3.15.1 to
 3.15.2 in /rootfs (#137)

Bumps [djangorestframework](https://github.com/encode/django-rest-framework) from 3.15.1 to 3.15.2.
- [Release notes](https://github.com/encode/django-rest-framework/releases)
- [Commits](https://github.com/encode/django-rest-framework/compare/3.15.1...3.15.2)

---
updated-dependencies:
- dependency-name: djangorestframework
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index ff5f4e6b6..22ad6e9e4 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -6,7 +6,7 @@ channels==4.1.0
 aiohttp==3.9.4
 django-cors-headers==4.3.1
 django-guardian==2.4.0
-djangorestframework==3.15.1
+djangorestframework==3.15.2
 docker==7.0.0
 gunicorn==22.0.0
 uvicorn[standard]==0.29.0

From fe79b9973324cbc865116a484c23f25cb25c8cc0 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 1 Jul 2024 17:25:00 +0800
Subject: [PATCH 032/200] feat(canary): remove canary api

---
 ...canaries_remove_release_canary_and_more.py |   34 +
 rootfs/api/models/app.py                      |   87 +-
 rootfs/api/models/appsettings.py              |   34 +-
 rootfs/api/models/build.py                    |    2 -
 rootfs/api/models/gateway.py                  |   42 +-
 rootfs/api/models/release.py                  |    6 +-
 rootfs/api/models/service.py                  |   24 +-
 rootfs/api/serializers/__init__.py            |   13 +-
 rootfs/api/serializers/schemas/dryccfile.py   |    2 +-
 rootfs/api/serializers/schemas/rules.py       | 2518 +++++++++++++++--
 rootfs/api/settings/production.py             |    1 -
 rootfs/api/signals.py                         |   25 +-
 rootfs/api/tests/rules/grpc_rules.json        |   67 +-
 rootfs/api/tests/rules/http_rules.json        |  104 +-
 rootfs/api/tests/rules/tcp_rules.json         |   22 +-
 rootfs/api/tests/test_app_settings.py         |   38 -
 rootfs/api/tests/test_build.py                |    7 +-
 rootfs/api/tests/test_canary.py               |   90 -
 rootfs/api/tests/test_gateway.py              |   92 +-
 rootfs/api/tests/test_procfile.py             |   10 +-
 rootfs/api/tests/test_release.py              |    4 +-
 rootfs/api/tests/test_service.py              |   23 -
 rootfs/api/urls.py                            |   10 +-
 rootfs/api/views.py                           |   44 +-
 rootfs/scheduler/tests/test_gateway.py        |   15 -
 25 files changed, 2557 insertions(+), 757 deletions(-)
 create mode 100644 rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py
 delete mode 100644 rootfs/api/tests/test_canary.py

diff --git a/rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py b/rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py
new file mode 100644
index 000000000..563a1e0d1
--- /dev/null
+++ b/rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py
@@ -0,0 +1,34 @@
+# Generated by Django 4.2.11 on 2024-07-01 07:58
+
+from django.db import migrations
+
+
+def migration_route_rules(apps, schema_editor):
+    route_model = apps.get_model('api', 'Route')
+    for route in route_model.objects.all():
+        if isinstance(route.rules, dict):
+            route.rules = route.rules.get("stable")
+        route.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_alter_domain_procfile_type_alter_route_procfile_type_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='appsettings',
+            name='canaries',
+        ),
+        migrations.RemoveField(
+            model_name='release',
+            name='canary',
+        ),
+        migrations.RemoveField(
+            model_name='service',
+            name='canary',
+        ),
+        migrations.RunPython(migration_route_rules),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 0aa664f02..576df08ba 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -42,7 +42,7 @@ def validate_app_id(value):
     """
     Check that the value follows the kubernetes name constraints
     """
-    match = re.match(r'[a-z]([a-z0-9-]*[a-z0-9])?(?<!-canary)$', value)
+    match = re.match(r'[a-z]([a-z0-9-]*[a-z0-9])?$', value)
     if not match:
         raise ValidationError("App name must start with an alphabetic character, cannot end with a"
                               + " hyphen and can only contain a-z (lowercase), 0-9 and hyphens.")
@@ -190,11 +190,8 @@ def restart(self, **kwargs):  # noqa
          Restart deployments with the kubectl rollout api
         """
         deployments = []
-        app_settings = self.appsettings_set.latest()
         if self.structure[kwargs['type']] > 0:
-            if kwargs['type'] in app_settings.canaries:
-                deployments.append(self._get_job_id(kwargs['type'], True))
-            deployments.append(self._get_job_id(kwargs['type'], False))
+            deployments.append(self._get_job_id(kwargs['type']))
         try:
             tasks = [
                 functools.partial(
@@ -219,13 +216,6 @@ def scale(self, user, structure):
             self.log(err_msg, logging.WARNING)
             raise DryccException(err_msg)
         app_settings = self.appsettings_set.latest()
-        if release.canary:
-            self._scale(
-                user,
-                structure,
-                self.release_set.filter(failed=False, canary=False).latest(),
-                app_settings
-            )
         self._scale(user, structure, release, app_settings)
 
     def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
@@ -293,9 +283,8 @@ def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_f
             if procfile_types is not None and scale_type not in procfile_types:
                 continue
             scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
-            if not release.canary or scale_type in app_settings.canaries:
-                deploys[scale_type] = self._gather_app_settings(
-                    release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
+            deploys[scale_type] = self._gather_app_settings(
+                release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
         self._deploy(
             deploys, procfile_types, prev_release, release, force_deploy, rollback_on_failure)
         # cleanup old release objects from kubernetes
@@ -307,22 +296,12 @@ def mount(self, user, volume, structure=None):
             raise DryccException('No build associated with this release')
         release = self.release_set.filter(failed=False).latest()
         app_settings = self.appsettings_set.latest()
-        if release.canary:
-            self._mount(
-                user,
-                volume,
-                self.release_set.filter(failed=False, canary=False).latest(),
-                app_settings,
-                structure=structure,
-            )
         self._mount(user, volume, release, app_settings, structure=structure)
 
     def cleanup_old(self, procfile_types=None):
-        names, app_settings = [], self.appsettings_set.latest()
+        names = []
         for scale_type in self.structure.keys():
-            if scale_type in app_settings.canaries:
-                names.append(self._get_job_id(scale_type, True))
-            names.append(self._get_job_id(scale_type, False))
+            names.append(self._get_job_id(scale_type))
         labels = {'heritage': 'drycc'}
         if procfile_types is not None:
             labels["type__in"] = procfile_types
@@ -360,7 +339,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         data['restart_policy'] = 'Never'
         data['active_deadline_seconds'] = timeout
         data['ttl_seconds_after_finished'] = expires
-        name = self._get_job_id(PROCFILE_TYPE_RUN, release.canary) + '-' + pod_name()
+        name = self._get_job_id(PROCFILE_TYPE_RUN) + '-' + pod_name()
         self.log("{} on {} runs '{}'".format(user.username, name, command))
         kwargs.update(data)
         try:
@@ -509,27 +488,21 @@ def image_pull_secret(self, namespace, registry, image):
         return name
 
     def state_to_k8s(self):
-        def _load_procfile_types(canary):
-            procfile_types = set()
-            for procfile_type, scale in self.structure.items():
-                response = self.scheduler().deployment.get(
-                    self.id, self._get_job_id(procfile_type, canary),
-                    ignore_exception=True)
-                if response.status_code == 404 and scale > 0:
-                    procfile_types.add(procfile_type)
-                elif response.status_code != 200:
-                    data = response.json()
-                    self.log('get deployment status_code {}, message: {}'.format(
-                        response.status_code, data.get("message", "")), logging.ERROR)
-            return procfile_types
-
         release = self.release_set.filter(failed=False).latest()
         if release.build is None:
             self.log('the last release does not have a build, skipping deployment...')
             return
-        procfile_types = _load_procfile_types(False)
-        if release.canary:
-            procfile_types = procfile_types.union(_load_procfile_types(canary=True))
+        procfile_types = set()
+        for procfile_type, scale in self.structure.items():
+            response = self.scheduler().deployment.get(
+                self.id, self._get_job_id(procfile_type),
+                ignore_exception=True)
+            if response.status_code == 404 and scale > 0:
+                procfile_types.add(procfile_type)
+            elif response.status_code != 200:
+                data = response.json()
+                self.log('get deployment status_code {}, message: {}'.format(
+                    response.status_code, data.get("message", "")), logging.ERROR)
         if len(procfile_types) == 0:
             self.log('the cluster status is the latest, skipping deployment...')
             return
@@ -585,11 +558,8 @@ def to_measurements(self, timestamp: float):
     def __str__(self):
         return self.id
 
-    def _get_job_id(self, container_type, canary):
-        job_id = f"{self.id}-{container_type}"
-        if canary:
-            job_id = f"{job_id}-canary"
-        return job_id
+    def _get_job_id(self, container_type):
+        return f"{self.id}-{container_type}"
 
     def _clean_app_logs(self):
         """Delete application logs stored by the logger component"""
@@ -607,15 +577,14 @@ def _mount(self, user, volume, release, app_settings, structure=None):
         volumes = Volume.objects.filter(app=self)
         tasks = []
         for scale_type, replicas in structure.items() if structure else self.structure.items():
-            if scale_type != PROCFILE_TYPE_RUN and (
-                    not release.canary or scale_type in app_settings.canaries):
+            if scale_type != PROCFILE_TYPE_RUN:
                 replicas = self.structure.get(scale_type, 0)
                 scale_type_volumes = [
                     volume for volume in volumes if scale_type in volume.path.keys()]
                 data = self._gather_app_settings(
                     release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
                 deployment = self.scheduler().deployment.get(
-                    self.id, self._get_job_id(scale_type, release.canary)).json()
+                    self.id, self._get_job_id(scale_type)).json()
                 spec_annotations = deployment['spec']['template']['metadata'].get(
                     'annotations', {})
                 self.set_application_config(release, scale_type)
@@ -623,7 +592,7 @@ def _mount(self, user, volume, release, app_settings, structure=None):
                 tasks.append(functools.partial(
                     self.scheduler().deployment.patch,
                     namespace=self.id,
-                    name=self._get_job_id(scale_type, release.canary),
+                    name=self._get_job_id(scale_type),
                     image=release.get_deploy_image(scale_type),
                     command=release.get_deploy_command(scale_type),
                     args=release.get_deploy_args(scale_type),
@@ -653,7 +622,7 @@ def _deploy(self, deploys, procfile_types, prev_release,
                 tasks.append(functools.partial(
                     self.scheduler().deploy,
                     namespace=self.id,
-                    name=self._get_job_id(scale_type, release.canary),
+                    name=self._get_job_id(scale_type),
                     image=release.get_deploy_image(scale_type),
                     command=release.get_deploy_command(scale_type),
                     args=release.get_deploy_args(scale_type),
@@ -664,8 +633,7 @@ def _deploy(self, deploys, procfile_types, prev_release,
             except KubeException as e:
                 # Don't rollback if the previous release doesn't have a build which means
                 # this is the first build and all the previous releases are just config changes.
-                if (prev_release.canary == release.canary and
-                        rollback_on_failure and prev_release.build is not None):
+                if (rollback_on_failure and prev_release.build is not None):
                     err = 'There was a problem deploying {}. Rolling back to release {}.'.format(
                         release.version_name, prev_release.version_name)
                     # This goes in the log before the rollback starts
@@ -701,7 +669,6 @@ def _scale(self, user, structure, release, app_settings):  # noqa
         """Scale containers up or down to match requested structure."""
         # use create to make sure minimum resources are created
         self.create()
-
         # Validate structure
         try:
             for target, count in structure.copy().items():
@@ -752,7 +719,7 @@ def _scale_pods(self, scale_types, release, app_settings):
                 functools.partial(
                     self.scheduler().scale,
                     namespace=self.id,
-                    name=self._get_job_id(scale_type, release.canary),
+                    name=self._get_job_id(scale_type),
                     image=release.get_deploy_image(scale_type),
                     command=release.get_deploy_command(scale_type),
                     args=release.get_deploy_args(scale_type),
@@ -905,7 +872,7 @@ def _check_deployment_in_progress(self, deploys, release, force_deploy=False):
         if force_deploy:
             return
         for scale_type, kwargs in deploys.items():
-            name = self._get_job_id(scale_type, release.canary)
+            name = self._get_job_id(scale_type)
             # Is there an existing deployment in progress?
             in_progress, deploy_okay = self.scheduler().deployment.in_progress(
                 self.id, name, kwargs.get("deploy_timeout"), kwargs.get("deploy_batches"),
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index 1021e4ae6..94f658257 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -19,7 +19,6 @@ class AppSettings(UuidAuditedModel):
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    canaries = models.JSONField(default=list)
     routable = models.BooleanField(default=True)
     autoscale = models.JSONField(default=dict, blank=True)
     label = models.JSONField(default=dict, blank=True)
@@ -63,20 +62,6 @@ def previous(self):
             prev_app_settings = None
         return prev_app_settings
 
-    def _update_canaries(self, previous_settings):
-        old = getattr(previous_settings, 'canaries', [])
-        new = getattr(self, 'canaries', [])
-        data = old.copy()
-        if data and not new:
-            setattr(self, 'canaries', data)
-        elif data != new:
-            for procfile_type in new:
-                if procfile_type not in data:
-                    data.append(procfile_type)
-            setattr(self, 'canaries', data)
-            self.summary += [
-                "{} add canaries for process types {}".format(self.owner, ','.join(new))]
-
     def _update_routable(self, previous_settings):
         old = getattr(previous_settings, 'routable', None)
         new = getattr(self, 'routable', None)
@@ -164,7 +149,7 @@ def _update_fields(self, ignore_update_fields=None):
             previous_settings = self.app.appsettings_set.latest()
         except AppSettings.DoesNotExist:
             pass
-        update_fields = ["canaries", "routable", "autoscale", "label"]
+        update_fields = ["routable", "autoscale", "label"]
         try:
             for update_field in update_fields:
                 if ignore_update_fields is None or update_field not in ignore_update_fields:
@@ -181,23 +166,6 @@ def _update_fields(self, ignore_update_fields=None):
         summary = ' '.join(self.summary)
         self.log('summary of app setting changes: {}'.format(summary), logging.DEBUG)
 
-    def diff_canaries(self):
-        prev_app_settings = self.previous()
-        action, canaries = None, []
-        if prev_app_settings is not None:
-            if prev_app_settings.canaries != self.canaries:
-                for procfile_type in self.canaries:  # add canary
-                    if procfile_type not in prev_app_settings.canaries:
-                        if action is None:
-                            action = "append"
-                        canaries.append(procfile_type)
-                for procfile_type in prev_app_settings.canaries:  # delete canary
-                    if procfile_type not in self.canaries:
-                        if action is None:
-                            action = "remove"
-                        canaries.append(procfile_type)
-        return prev_app_settings, action, canaries
-
     @transaction.atomic
     def save(self, ignore_update_field=None, *args, **kwargs):
         self._update_fields(ignore_update_field)
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 32df3a48b..e8bb99d43 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -71,7 +71,6 @@ def get_image(self, procfile_type, default_image=None):
         return default_image if default_image else self.image
 
     def create_release(self, user, *args, **kwargs):
-        app_settings = self.app.appsettings_set.latest()
         latest_release = self.app.release_set.filter(failed=False).latest()
         latest_version = self.app.release_set.latest().version
         try:
@@ -79,7 +78,6 @@ def create_release(self, user, *args, **kwargs):
                 user,
                 build=self,
                 config=latest_release.config,
-                canary=len(app_settings.canaries) > 0,
             )
             run_pipeline.delay(new_release)
             return new_release
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index d71a5592e..911a6b6f8 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -1,4 +1,3 @@
-import re
 import logging
 from django.db import models
 from django.conf import settings
@@ -201,35 +200,16 @@ def hostnames(self):
     @property
     def default_rules(self):
         service = get_object_or_404(self.app.service_set, procfile_type=self.procfile_type)
-        stable_backend_refs, canary_backend_refs = [], []
+        backend_refs = []
         for item in service.ports:
             if item["port"] == self.port:
-                stable_backend = {
+                backend_refs.append({
                     "kind": "Service",
                     "name": str(service),
                     "port": item["port"],
                     "weight": 100,
-                }
-                canary_backend = {
-                    "kind": "Service",
-                    "name": "%s-canary" % str(service),
-                    "port": item["port"],
-                    "weight": 0,
-                }
-                stable_backend_refs.append(stable_backend)
-                canary_backend_refs.append(stable_backend)
-                canary_backend_refs.append(canary_backend)
-        return {
-            "stable": [{"backendRefs": stable_backend_refs}],
-            "canary": [{"backendRefs": canary_backend_refs}],
-        }
-
-    @property
-    def current_rules(self):
-        app_settings = self.app.appsettings_set.latest()
-        if self.procfile_type in app_settings.canaries:
-            return self.rules["canary"]
-        return self.rules["stable"]
+                })
+        return [{"backendRefs": backend_refs}]
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this service.
@@ -246,13 +226,11 @@ def check_rules(self):
         service = self.app.service_set.filter(
             procfile_type=self.procfile_type).first()
         ports = [item["port"] for item in service.ports]
-        for rules in self.rules.values():
-            for rule in rules:
-                for backend_ref in rule["backendRefs"]:
-                    port = backend_ref["port"]
-                    name = re.sub(r'(.*)-canary', r'\1', backend_ref["name"])
-                    if port not in ports or name != str(service):
-                        return False, {"detail": "backendRefs associated with incorrect service"}
+        for rule in self.rules:
+            for backend_ref in rule["backendRefs"]:
+                port = backend_ref["port"]
+                if port not in ports or backend_ref["name"] != str(service):
+                    return False, {"detail": "backendRefs associated with incorrect service"}
         return True, ""
 
     def refresh_to_k8s(self):
@@ -266,7 +244,7 @@ def refresh_to_k8s(self):
                 self.scheduler().httproute.delete(self.app.id, self._https_redirect_name)
             else:
                 parent_refs.extend(http_parent_refs)
-            self._refresh_to_k8s(self.current_rules, parent_refs)
+            self._refresh_to_k8s(self.rules, parent_refs)
         else:
             self.scheduler().httproute.delete(self.app.id, self.name)
             self.scheduler().httproute.delete(self.app.id, self._https_redirect_name)
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index fedf74eb3..557486d8b 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -32,7 +32,6 @@ class Release(UuidAuditedModel):
     version = models.PositiveIntegerField()
     summary = models.TextField(blank=True, null=True)
     failed = models.BooleanField(default=False)
-    canary = models.BooleanField(default=False)
     exception = models.TextField(blank=True, null=True)
 
     config = models.ForeignKey('Config', on_delete=models.CASCADE)
@@ -148,7 +147,7 @@ def log(self, message, level=logging.INFO):
         """
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
-    def new(self, user, config, build, summary=None, canary=False):
+    def new(self, user, config, build, summary=None):
         """
         Create a new application release using the provided Build and Config
         on behalf of a user.
@@ -160,7 +159,7 @@ def new(self, user, config, build, summary=None, canary=False):
         # create new release and auto-increment version
         return Release.objects.create(
             owner=user, app=self.app, config=config,
-            build=build, version=new_version, summary=summary, canary=canary
+            build=build, version=new_version, summary=summary
         )
 
     def get_port(self, procfile_type):
@@ -210,7 +209,6 @@ def rollback(self, user, version=None):
                 build=prev.build,
                 config=prev.config,
                 summary="{} rolled back to v{}".format(user, version),
-                canary=len(app_settings.canaries) > 0,
             )
 
             if self.build is not None:
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index 3d44fa1b0..82758dd54 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -33,7 +33,6 @@ class Service(AuditedModel):
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     ports = models.JSONField(
         default=list, validators=[partial(validate_json, schema=service_ports_schema)])
-    canary = models.BooleanField(default=False)
     procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
 
     class Meta:
@@ -42,19 +41,18 @@ class Meta:
         ordering = ['-created']
 
     def __str__(self):
-        return self._svc_name(False)
+        return self._svc_name()
 
     @property
     def domain(self):
         return "{}.{}.svc.{}".format(
-            self._svc_name(False), self._namespace(), settings.KUBERNETES_CLUSTER_DOMAIN
+            self._svc_name(), self._namespace(), settings.KUBERNETES_CLUSTER_DOMAIN
         )
 
     def as_dict(self):
         return {
             "domain": self.domain,
             "ports": self.ports,
-            "canary": self.canary,
             "procfile_type": self.procfile_type,
         }
 
@@ -97,22 +95,13 @@ def remove_port(self, port, protocol):
             return True
         return False
 
-    def refresh_k8s_svc(self):
-        if self.canary:
-            self._refresh_k8s_svc(self._svc_name(False))
-        else:
-            self._delete_k8s_svc(self._svc_name(True))
-        self._refresh_k8s_svc(self._svc_name(self.canary))
-
     def save(self, *args, **kwargs):
         service = super(Service, self).save(*args, **kwargs)
         self.refresh_k8s_svc()
         return service
 
     def delete(self, *args, **kwargs):
-        if self.canary:
-            self._delete_k8s_svc(self._svc_name(False))
-        self._delete_k8s_svc(self._svc_name(self.canary))
+        self._delete_k8s_svc(self._svc_name())
         # Delete from DB
         return super(Service, self).delete(*args, **kwargs)
 
@@ -130,16 +119,15 @@ def log(self, message, level=logging.INFO):
     def _namespace(self):
         return self.app.id
 
-    def _svc_name(self, canary):
+    def _svc_name(self):
         if self.procfile_type == 'web':
             svc_name = self.app.id
         else:
             svc_name = "{}-{}".format(self.app.id, self.procfile_type)
-        if canary:
-            svc_name = "%s-canary" % svc_name
         return svc_name
 
-    def _refresh_k8s_svc(self, svc_name):
+    def refresh_k8s_svc(self):
+        svc_name = self._svc_name()
         namespace = self._namespace()
         self.log('creating service: {}'.format(svc_name), level=logging.DEBUG)
         try:
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index e580ad355..b0ee8dafc 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -19,7 +19,7 @@
 from api.exceptions import DryccException
 from api.models.base import PROCFILE_TYPE_MIN_LENGTH, PROCFILE_TYPE_MAX_LENGTH
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
-from .schemas.rules import SCHEMA as RULES_SCHEMA
+from .schemas import rules
 from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
 from .schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
 from .schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
@@ -545,7 +545,6 @@ class AppSettingsSerializer(serializers.ModelSerializer):
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     owner = serializers.ReadOnlyField(source='owner.username')
-    canaries = serializers.JSONField(required=False)
     autoscale = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
     label = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
 
@@ -746,6 +745,10 @@ def validate_kind(value):
 
     validate_procfile_type = staticmethod(validate_procfile_type)
 
-    @staticmethod
-    def validate_rules(value):
-        return validate_json(value, RULES_SCHEMA, serializers.ValidationError)
+    def validate_rules(self, value):
+        kind = getattr(self, "initial_data", getattr(self, "data", {})).get("kind", None)
+        if kind:
+            schema = getattr(rules, f"{kind.replace("Route", "")}_RULES_SCHEMA", "SCHEMA")
+        else:
+            schema = rules.SCHEMA
+        return validate_json(value, schema, serializers.ValidationError)
diff --git a/rootfs/api/serializers/schemas/dryccfile.py b/rootfs/api/serializers/schemas/dryccfile.py
index 0c956aa72..3ad8efab8 100644
--- a/rootfs/api/serializers/schemas/dryccfile.py
+++ b/rootfs/api/serializers/schemas/dryccfile.py
@@ -1,4 +1,4 @@
-PROCTYPE_REGEX = r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)(?<!-canary)$'
+PROCTYPE_REGEX = r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)$'
 
 SCHEMA = {
     "$schema": "http://json-schema.org/schema#",
diff --git a/rootfs/api/serializers/schemas/rules.py b/rootfs/api/serializers/schemas/rules.py
index b9b10f254..e12b6ae3c 100644
--- a/rootfs/api/serializers/schemas/rules.py
+++ b/rootfs/api/serializers/schemas/rules.py
@@ -1,215 +1,2417 @@
-HTTP_HEADER_FILTER_SCHEMA = {
-    "set": {
-        "type": "array",
-        "items": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string"},
-                "value": {"type": "string"}
-            }
-        }
-    },
-    "add": {
-        "type": "array",
-        "items": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string"},
-                "value": {"type": "string"}
+# flake8: noqa
+
+
+TCP_RULES_SCHEMA = {
+    "description": "Rules are a list of TCP matchers and actions.",
+    "items": {
+        "description": "TCPRouteRule is the configuration for a given rule.",
+        "properties": {
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent. If unspecified or invalid (refers to a non-existent resource or a\nService with no endpoints), the underlying implementation MUST actively\nreject connection attempts to this backend. Connection rejections must\nrespect weight; if an invalid backend is requested to have 80% of\nconnections, then 80% of connections must be rejected instead.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Extended for Kubernetes ServiceImport\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Extended",
+                "items": {
+                    "description": "BackendRef defines how a Route should forward a request to a Kubernetes\nresource.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>\n\n\nNote that when the BackendTLSPolicy object is enabled by the implementation,\nthere are some extra rules about validity to consider here. See the fields\nwhere this struct is used for more information about the exact behavior.",
+                    "properties": {
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "minItems": 1,
+                "type": "array"
             }
-        }
+        },
+        "type": "object"
     },
-    "remove": {"type": "array"}
+    "maxItems": 16,
+    "minItems": 1,
+    "type": "array"
 }
 
-FILTER_SCHEMA = {
-    # Type identifies the type of filter to apply.
-    # As with other API fields, types are classified into three conformance levels:
-    "type": {
-        "type": "string",
-        "enum": ["ExtensionRef", "RequestHeaderModifier", "RequestMirror", "RequestRedirect", "ResponseHeaderModifier", "URLRewrite"],  # noqa
-    },
-    # ExtensionRef is an optional, implementation-specific extension to the “filter” behavior. # noqa
-    # For example, resource “myroutefilter” in group “networking.example.net”).
-    # ExtensionRef MUST NOT be used for core and extended filters.
-    "extensionRef": {
-        "type": "object",
+UDP_RULES_SCHEMA = {
+    "description": "Rules are a list of UDP matchers and actions.",
+    "items": {
+        "description": "UDPRouteRule is the configuration for a given rule.",
         "properties": {
-            "group": {"type": "string"},
-            "kind": {"type": "string"},
-            "name": {"type": "string"}
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent. If unspecified or invalid (refers to a non-existent resource or a\nService with no endpoints), the underlying implementation MUST actively\nreject connection attempts to this backend. Packet drops must\nrespect weight; if an invalid backend is requested to have 80% of\nthe packets, then 80% of packets must be dropped instead.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Extended for Kubernetes ServiceImport\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Extended",
+                "items": {
+                    "description": "BackendRef defines how a Route should forward a request to a Kubernetes\nresource.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>\n\n\nNote that when the BackendTLSPolicy object is enabled by the implementation,\nthere are some extra rules about validity to consider here. See the fields\nwhere this struct is used for more information about the exact behavior.",
+                    "properties": {
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "minItems": 1,
+                "type": "array"
+            }
         },
-        "required": ["group", "kind", "name"],
-        "additionalProperties": False
-    },
-    # RequestHeaderModifier defines a schema for a filter that modifies request headers.
-    "requestHeaderModifier": {
-        "type": "object",
-        "properties": HTTP_HEADER_FILTER_SCHEMA,
-        "additionalProperties": False
+        "type": "object"
     },
-    # ResponseHeaderModifier defines a schema for a filter that modifies response headers.
-    "responseHeaderModifier": {
-        "type": "object",
-        "properties": HTTP_HEADER_FILTER_SCHEMA,
-        "additionalProperties": False
-    },
-    # RequestMirror defines a schema for a filter that mirrors requests.
-    # Requests are sent to the specified destination, but responses from that destination are ignored. # noqa
-    "requestMirror": {
-        "type": "object",
+    "maxItems": 16,
+    "minItems": 1,
+    "type": "array"
+}
+
+TLS_RULES_SCHEMA = {
+    "description": "Rules are a list of TLS matchers and actions.",
+    "items": {
+        "description": "TLSRouteRule is the configuration for a given rule.",
         "properties": {
-            "backendRef": {
-                "properties": {
-                    "group": {"type": "string"},
-                    "kind": {"type": "string"},
-                    "name": {"type": "string"},
-                    "namespace": {"type": "string"},
-                    "port": {"type": "integer"},
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent. If unspecified or invalid (refers to a non-existent resource or\na Service with no endpoints), the rule performs no forwarding; if no\nfilters are specified that would result in a response being sent, the\nunderlying implementation must actively reject request attempts to this\nbackend, by rejecting the connection or returning a 500 status code.\nRequest rejections must respect weight; if an invalid backend is\nrequested to have 80% of requests, then 80% of requests must be rejected\ninstead.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Extended for Kubernetes ServiceImport\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Extended",
+                "items": {
+                    "description": "BackendRef defines how a Route should forward a request to a Kubernetes\nresource.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>\n\n\nNote that when the BackendTLSPolicy object is enabled by the implementation,\nthere are some extra rules about validity to consider here. See the fields\nwhere this struct is used for more information about the exact behavior.",
+                    "properties": {
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
                 },
-                "required": ["name"],
-                "additionalProperties": False
-            },
+                "maxItems": 16,
+                "minItems": 1,
+                "type": "array"
+            }
         },
-        "required": ["backendRef"],
-        "additionalProperties": False
-    },
-    # RequestRedirect defines a schema for a filter that responds to the request with an HTTP redirection. # noqa
-    "requestRedirect": {
-        "type": "object",
-        "properties": {
-            "scheme": {"type": "string"},
-            "hostname": {"type": "string"},
-            "path": {"type": "string"},
-            "port": {"type": "integer"},
-            "statusCode": {"type": "integer"}
-        }
+        "type": "object"
     },
-    # URLRewrite defines a schema for a filter that modifies a request during forwarding
-    "urlRewrite": {
-        "hostname": {"type": "string"},
-        "path": {"type": "string"},
-    }
+    "maxItems": 16,
+    "minItems": 1,
+    "type": "array"
 }
 
-RULES_SCHEMA = {
-    "type": "array",
+HTTP_RULES_SCHEMA = {
+    "description": "Rules are a list of HTTP matchers, filters and actions.",
     "items": {
         "properties": {
-            # Matches define conditions used for matching the rule against incoming HTTP requests. # noqa
-            # Each match is independent, i.e. this rule will be matched if any one of the matches is satisfied. # noqa
-            # More info: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteMatch # noqa
-            "matches": {
+            "backendRefs": {
+                "items": {
+                    "properties": {
+                        "filters": {
+                            "items": {
+                                "properties": {
+                                    "extensionRef": {
+                                        "properties": {
+                                            "group": {
+                                                "description": "",
+                                                "maxLength": 253,
+                                                "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "kind": {
+                                                "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                                "maxLength": 63,
+                                                "minLength": 1,
+                                                "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                "type": "string"
+                                            },
+                                            "name": {
+                                                "description": "Name is the name of the referent.",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "group",
+                                            "kind",
+                                            "name"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "requestHeaderModifier": {
+                                        "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "requestMirror": {
+                                        "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "backendRef": {
+                                                "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                                "properties": {
+                                                    "group": {
+                                                        "default": "",
+                                                        "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                                        "maxLength": 253,
+                                                        "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                        "type": "string"
+                                                    },
+                                                    "kind": {
+                                                        "default": "Service",
+                                                        "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "name": {
+                                                        "description": "Name is the name of the referent.",
+                                                        "maxLength": 253,
+                                                        "minLength": 1,
+                                                        "type": "string"
+                                                    },
+                                                    "namespace": {
+                                                        "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "port": {
+                                                        "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                                        "format": "int32",
+                                                        "maximum": 65535,
+                                                        "minimum": 1,
+                                                        "type": "integer"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "name"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "Must have port for Service reference",
+                                                        "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                                    }
+                                                ]
+                                            }
+                                        },
+                                        "required": [
+                                            "backendRef"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "requestRedirect": {
+                                        "description": "RequestRedirect defines a schema for a filter that responds to the\nrequest with an HTTP redirection.\n\n\nSupport: Core",
+                                        "properties": {
+                                            "hostname": {
+                                                "description": "Hostname is the hostname to be used in the value of the `Location`\nheader in the response.\nWhen empty, the hostname in the `Host` header of the request is used.\n\n\nSupport: Core",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "path": {
+                                                "description": "Path defines parameters used to modify the path of the incoming request.\nThe modified path is then used to construct the `Location` header. When\nempty, the request path is used as-is.\n\n\nSupport: Extended",
+                                                "properties": {
+                                                    "replaceFullPath": {
+                                                        "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "replacePrefixMatch": {
+                                                        "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "type": {
+                                                        "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                                        "enum": [
+                                                            "ReplaceFullPath",
+                                                            "ReplacePrefixMatch"
+                                                        ],
+                                                        "type": "string"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "type"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                                        "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                                        "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                                    },
+                                                    {
+                                                        "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                                        "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                                        "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                                    }
+                                                ]
+                                            },
+                                            "port": {
+                                                "description": "Port is the port to be used in the value of the `Location`\nheader in the response.\n\n\nIf no port is specified, the redirect port MUST be derived using the\nfollowing rules:\n\n\n* If redirect scheme is not-empty, the redirect port MUST be the well-known\n  port associated with the redirect scheme. Specifically \"http\" to port 80\n  and \"https\" to port 443. If the redirect scheme does not have a\n  well-known port, the listener port of the Gateway SHOULD be used.\n* If redirect scheme is empty, the redirect port MUST be the Gateway\n  Listener port.\n\n\nImplementations SHOULD NOT add the port number in the 'Location'\nheader in the following cases:\n\n\n* A Location header that will use HTTP (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 80.\n* A Location header that will use HTTPS (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 443.\n\n\nSupport: Extended",
+                                                "format": "int32",
+                                                "maximum": 65535,
+                                                "minimum": 1,
+                                                "type": "integer"
+                                            },
+                                            "scheme": {
+                                                "description": "Scheme is the scheme to be used in the value of the `Location` header in\nthe response. When empty, the scheme of the request is used.\n\n\nScheme redirects can affect the port of the redirect, for more information,\nrefer to the documentation for the port field of this filter.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Extended",
+                                                "enum": [
+                                                    "http",
+                                                    "https"
+                                                ],
+                                                "type": "string"
+                                            },
+                                            "statusCode": {
+                                                "default": 302,
+                                                "description": "StatusCode is the HTTP status code to be used in response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Core",
+                                                "enum": [
+                                                    301,
+                                                    302
+                                                ],
+                                                "type": "integer"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "responseHeaderModifier": {
+                                        "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "type": {
+                                        "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations must support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by\n  specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` should be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                        "enum": [
+                                            "RequestHeaderModifier",
+                                            "ResponseHeaderModifier",
+                                            "RequestMirror",
+                                            "RequestRedirect",
+                                            "URLRewrite",
+                                            "ExtensionRef"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "urlRewrite": {
+                                        "description": "URLRewrite defines a schema for a filter that modifies a request during forwarding.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "hostname": {
+                                                "description": "Hostname is the value to be used to replace the Host header value during\nforwarding.\n\n\nSupport: Extended",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "path": {
+                                                "description": "Path defines a path rewrite.\n\n\nSupport: Extended",
+                                                "properties": {
+                                                    "replaceFullPath": {
+                                                        "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "replacePrefixMatch": {
+                                                        "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "type": {
+                                                        "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                                        "enum": [
+                                                            "ReplaceFullPath",
+                                                            "ReplacePrefixMatch"
+                                                        ],
+                                                        "type": "string"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "type"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                                        "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                                        "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                                    },
+                                                    {
+                                                        "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                                        "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                                        "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                                    }
+                                                ]
+                                            }
+                                        },
+                                        "type": "object"
+                                    }
+                                },
+                                "required": [
+                                    "type"
+                                ],
+                                "type": "object",
+                                "x-kubernetes-validations": [
+                                    {
+                                        "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                                        "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                                        "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                                        "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                                        "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                                        "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                                        "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.requestRedirect must be nil if the filter.type is not RequestRedirect",
+                                        "rule": "!(has(self.requestRedirect) && self.type != 'RequestRedirect')"
+                                    },
+                                    {
+                                        "message": "filter.requestRedirect must be specified for RequestRedirect filter.type",
+                                        "rule": "!(!has(self.requestRedirect) && self.type == 'RequestRedirect')"
+                                    },
+                                    {
+                                        "message": "filter.urlRewrite must be nil if the filter.type is not URLRewrite",
+                                        "rule": "!(has(self.urlRewrite) && self.type != 'URLRewrite')"
+                                    },
+                                    {
+                                        "message": "filter.urlRewrite must be specified for URLRewrite filter.type",
+                                        "rule": "!(!has(self.urlRewrite) && self.type == 'URLRewrite')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                                        "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                                        "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                                    }
+                                ]
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both",
+                                    "rule": "!(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite'))"
+                                },
+                                {
+                                    "message": "May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both",
+                                    "rule": "!(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite'))"
+                                },
+                                {
+                                    "message": "RequestHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                                },
+                                {
+                                    "message": "ResponseHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                                },
+                                {
+                                    "message": "RequestRedirect filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'RequestRedirect').size() <= 1"
+                                },
+                                {
+                                    "message": "URLRewrite filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'URLRewrite').size() <= 1"
+                                }
+                            ]
+                        },
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "type": "array"
+            },
+            "filters": {
+                "description": "Filters define the filters that are applied to requests that match\nthis rule.\n\n\nWherever possible, implementations SHOULD implement filters in the order\nthey are specified.\n\n\nImplementations MAY choose to implement this ordering strictly, rejecting\nany combination or order of filters that can not be supported. If implementations\nchoose a strict interpretation of filter ordering, they MUST clearly document\nthat behavior.\n\n\nTo reject an invalid combination or order of filters, implementations SHOULD\nconsider the Route Rules with this configuration invalid. If all Route Rules\nin a Route are invalid, the entire Route would be considered invalid. If only\na portion of Route Rules are invalid, implementations MUST set the\n\"PartiallyInvalid\" condition for the Route.\n\n\nConformance-levels at this level are defined based on the type of filter:\n\n\n- ALL core filters MUST be supported by all implementations.\n- Implementers are encouraged to support extended filters.\n- Implementation-specific custom filters have no API guarantees across\n  implementations.\n\n\nSpecifying the same filter multiple times is not supported unless explicitly\nindicated in the filter.\n\n\nAll filters are expected to be compatible with each other except for the\nURLRewrite and RequestRedirect filters, which may not be combined. If an\nimplementation can not support other combinations of filters, they must clearly\ndocument that limitation. In cases where incompatible or unsupported\nfilters are specified and cause the `Accepted` condition to be set to status\n`False`, implementations may use the `IncompatibleFilters` reason to specify\nthis configuration error.\n\n\nSupport: Core",
+                "items": {
+                    "description": "HTTPRouteFilter defines processing steps that must be completed during the\nrequest or response lifecycle. HTTPRouteFilters are meant as an extension\npoint to express processing that may be done in Gateway implementations. Some\nexamples include request or response modification, implementing\nauthentication strategies, rate-limiting, and traffic shaping. API\nguarantee/conformance is defined based on the type of the filter.",
+                    "properties": {
+                        "extensionRef": {
+                            "description": "ExtensionRef is an optional, implementation-specific extension to the\n\"filter\" behavior.  For example, resource \"myroutefilter\" in group\n\"networking.example.net\"). ExtensionRef MUST NOT be used for core and\nextended filters.\n\n\nThis filter can be used multiple times within the same rule.\n\n\nSupport: Implementation-specific",
+                            "properties": {
+                                "group": {
+                                    "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                    "maxLength": 253,
+                                    "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "kind": {
+                                    "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                    "maxLength": 63,
+                                    "minLength": 1,
+                                    "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                    "type": "string"
+                                },
+                                "name": {
+                                    "description": "Name is the name of the referent.",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "group",
+                                "kind",
+                                "name"
+                            ],
+                            "type": "object"
+                        },
+                        "requestHeaderModifier": {
+                            "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "requestMirror": {
+                            "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                            "properties": {
+                                "backendRef": {
+                                    "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                    "properties": {
+                                        "group": {
+                                            "default": "",
+                                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                            "maxLength": 253,
+                                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                            "type": "string"
+                                        },
+                                        "kind": {
+                                            "default": "Service",
+                                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "name": {
+                                            "description": "Name is the name of the referent.",
+                                            "maxLength": 253,
+                                            "minLength": 1,
+                                            "type": "string"
+                                        },
+                                        "namespace": {
+                                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "port": {
+                                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                            "format": "int32",
+                                            "maximum": 65535,
+                                            "minimum": 1,
+                                            "type": "integer"
+                                        }
+                                    },
+                                    "required": [
+                                        "name"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "Must have port for Service reference",
+                                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                        }
+                                    ]
+                                }
+                            },
+                            "required": [
+                                "backendRef"
+                            ],
+                            "type": "object"
+                        },
+                        "requestRedirect": {
+                            "description": "RequestRedirect defines a schema for a filter that responds to the\nrequest with an HTTP redirection.\n\n\nSupport: Core",
+                            "properties": {
+                                "hostname": {
+                                    "description": "Hostname is the hostname to be used in the value of the `Location`\nheader in the response.\nWhen empty, the hostname in the `Host` header of the request is used.\n\n\nSupport: Core",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "path": {
+                                    "description": "Path defines parameters used to modify the path of the incoming request.\nThe modified path is then used to construct the `Location` header. When\nempty, the request path is used as-is.\n\n\nSupport: Extended",
+                                    "properties": {
+                                        "replaceFullPath": {
+                                            "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "replacePrefixMatch": {
+                                            "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "type": {
+                                            "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                            "enum": [
+                                                "ReplaceFullPath",
+                                                "ReplacePrefixMatch"
+                                            ],
+                                            "type": "string"
+                                        }
+                                    },
+                                    "required": [
+                                        "type"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                            "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                            "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                        },
+                                        {
+                                            "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                            "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                            "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                        }
+                                    ]
+                                },
+                                "port": {
+                                    "description": "Port is the port to be used in the value of the `Location`\nheader in the response.\n\n\nIf no port is specified, the redirect port MUST be derived using the\nfollowing rules:\n\n\n* If redirect scheme is not-empty, the redirect port MUST be the well-known\n  port associated with the redirect scheme. Specifically \"http\" to port 80\n  and \"https\" to port 443. If the redirect scheme does not have a\n  well-known port, the listener port of the Gateway SHOULD be used.\n* If redirect scheme is empty, the redirect port MUST be the Gateway\n  Listener port.\n\n\nImplementations SHOULD NOT add the port number in the 'Location'\nheader in the following cases:\n\n\n* A Location header that will use HTTP (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 80.\n* A Location header that will use HTTPS (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 443.\n\n\nSupport: Extended",
+                                    "format": "int32",
+                                    "maximum": 65535,
+                                    "minimum": 1,
+                                    "type": "integer"
+                                },
+                                "scheme": {
+                                    "description": "Scheme is the scheme to be used in the value of the `Location` header in\nthe response. When empty, the scheme of the request is used.\n\n\nScheme redirects can affect the port of the redirect, for more information,\nrefer to the documentation for the port field of this filter.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Extended",
+                                    "enum": [
+                                        "http",
+                                        "https"
+                                    ],
+                                    "type": "string"
+                                },
+                                "statusCode": {
+                                    "default": 302,
+                                    "description": "StatusCode is the HTTP status code to be used in response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Core",
+                                    "enum": [
+                                        301,
+                                        302
+                                    ],
+                                    "type": "integer"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "responseHeaderModifier": {
+                            "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "type": {
+                            "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations must support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by\n  specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` should be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                            "enum": [
+                                "RequestHeaderModifier",
+                                "ResponseHeaderModifier",
+                                "RequestMirror",
+                                "RequestRedirect",
+                                "URLRewrite",
+                                "ExtensionRef"
+                            ],
+                            "type": "string"
+                        },
+                        "urlRewrite": {
+                            "description": "URLRewrite defines a schema for a filter that modifies a request during forwarding.\n\n\nSupport: Extended",
+                            "properties": {
+                                "hostname": {
+                                    "description": "Hostname is the value to be used to replace the Host header value during\nforwarding.\n\n\nSupport: Extended",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "path": {
+                                    "description": "Path defines a path rewrite.\n\n\nSupport: Extended",
+                                    "properties": {
+                                        "replaceFullPath": {
+                                            "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "replacePrefixMatch": {
+                                            "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "type": {
+                                            "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                            "enum": [
+                                                "ReplaceFullPath",
+                                                "ReplacePrefixMatch"
+                                            ],
+                                            "type": "string"
+                                        }
+                                    },
+                                    "required": [
+                                        "type"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                            "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                            "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                        },
+                                        {
+                                            "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                            "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                            "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                        }
+                                    ]
+                                }
+                            },
+                            "type": "object"
+                        }
+                    },
+                    "required": [
+                        "type"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                            "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                            "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                            "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                            "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                            "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                            "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.requestRedirect must be nil if the filter.type is not RequestRedirect",
+                            "rule": "!(has(self.requestRedirect) && self.type != 'RequestRedirect')"
+                        },
+                        {
+                            "message": "filter.requestRedirect must be specified for RequestRedirect filter.type",
+                            "rule": "!(!has(self.requestRedirect) && self.type == 'RequestRedirect')"
+                        },
+                        {
+                            "message": "filter.urlRewrite must be nil if the filter.type is not URLRewrite",
+                            "rule": "!(has(self.urlRewrite) && self.type != 'URLRewrite')"
+                        },
+                        {
+                            "message": "filter.urlRewrite must be specified for URLRewrite filter.type",
+                            "rule": "!(!has(self.urlRewrite) && self.type == 'URLRewrite')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                            "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                            "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                        }
+                    ]
+                },
+                "maxItems": 16,
                 "type": "array",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both",
+                        "rule": "!(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite'))"
+                    },
+                    {
+                        "message": "RequestHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                    },
+                    {
+                        "message": "ResponseHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                    },
+                    {
+                        "message": "RequestRedirect filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'RequestRedirect').size() <= 1"
+                    },
+                    {
+                        "message": "URLRewrite filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'URLRewrite').size() <= 1"
+                    }
+                ]
+            },
+            "matches": {
+                "default": [
+                    {
+                        "path": {
+                            "type": "PathPrefix",
+                            "value": "/"
+                        }
+                    }
+                ],
+                "description": "Matches define conditions used for matching the rule against incoming\nHTTP requests. Each match is independent, i.e. this rule will be matched\nif **any** one of the matches is satisfied.\n\n\nFor example, take the following matches configuration:\n\n\n```\nmatches:\n- path:\n    value: \"/foo\"\n  headers:\n  - name: \"version\"\n    value: \"v2\"\n- path:\n    value: \"/v2/foo\"\n```\n\n\nFor a request to match against this rule, a request must satisfy\nEITHER of the two conditions:\n\n\n- path prefixed with `/foo` AND contains the header `version: v2`\n- path prefix of `/v2/foo`\n\n\nSee the documentation for HTTPRouteMatch on how to specify multiple\nmatch conditions that should be ANDed together.\n\n\nIf no matches are specified, the default is a prefix\npath match on \"/\", which has the effect of matching every\nHTTP request.\n\n\nProxy or Load Balancer routing configuration generated from HTTPRoutes\nMUST prioritize matches based on the following criteria, continuing on\nties. Across all rules specified on applicable Routes, precedence must be\ngiven to the match having:\n\n\n* \"Exact\" path match.\n* \"Prefix\" path match with largest number of characters.\n* Method match.\n* Largest number of header matches.\n* Largest number of query param matches.\n\n\nNote: The precedence of RegularExpression path matches are implementation-specific.\n\n\nIf ties still exist across multiple Routes, matching precedence MUST be\ndetermined in order of the following criteria, continuing on ties:\n\n\n* The oldest Route based on creation timestamp.\n* The Route appearing first in alphabetical order by\n  \"{namespace}/{name}\".\n\n\nIf ties still exist within an HTTPRoute, matching precedence MUST be granted\nto the FIRST matching rule (in list order) with a match meeting the above\ncriteria.\n\n\nWhen no rules matching a request have been successfully attached to the\nparent a request is coming from, a HTTP 404 status code MUST be returned.",
                 "items": {
+                    "description": "HTTPRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\n\nFor example, the match below will match a HTTP request only if its path\nstarts with `/foo` AND it contains the `version: v1` header:\n\n\n```\nmatch:\n\n\n\tpath:\n\t  value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t  value \"v1\"\n\n\n```",
                     "properties": {
-                        # Path specifies a HTTP request path matcher.
-                        # If this field is not specified, a default prefix match on the “/” path is provided. # noqa
+                        "headers": {
+                            "description": "Headers specifies HTTP request header matchers. Multiple match values are\nANDed together, meaning, a request must match all the specified headers\nto select the route.",
+                            "items": {
+                                "description": "HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request\nheaders.",
+                                "properties": {
+                                    "name": {
+                                        "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, only the first\nentry with an equivalent name MUST be considered for a match. Subsequent\nentries with an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.\n\n\nWhen a header is repeated in an HTTP request, it is\nimplementation-specific behavior as to how this is represented.\nGenerally, proxies should follow the guidance from the RFC:\nhttps://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding\nprocessing a repeated header, with special handling for \"Set-Cookie\".",
+                                        "maxLength": 256,
+                                        "minLength": 1,
+                                        "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                        "type": "string"
+                                    },
+                                    "type": {
+                                        "default": "Exact",
+                                        "description": "Type specifies how to match against the value of the header.\n\n\nSupport: Core (Exact)\n\n\nSupport: Implementation-specific (RegularExpression)\n\n\nSince RegularExpression HeaderMatchType has implementation-specific\nconformance, implementations can support POSIX, PCRE or any other dialects\nof regular expressions. Please read the implementation's documentation to\ndetermine the supported dialect.",
+                                        "enum": [
+                                            "Exact",
+                                            "RegularExpression"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "value": {
+                                        "description": "Value is the value of HTTP Header to be matched.",
+                                        "maxLength": 4096,
+                                        "minLength": 1,
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "name",
+                                    "value"
+                                ],
+                                "type": "object"
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-list-map-keys": [
+                                "name"
+                            ],
+                            "x-kubernetes-list-type": "map"
+                        },
+                        "method": {
+                            "description": "Method specifies HTTP method matcher.\nWhen specified, this route will be matched only if the request has the\nspecified method.\n\n\nSupport: Extended",
+                            "enum": [
+                                "GET",
+                                "HEAD",
+                                "POST",
+                                "PUT",
+                                "DELETE",
+                                "CONNECT",
+                                "OPTIONS",
+                                "TRACE",
+                                "PATCH"
+                            ],
+                            "type": "string"
+                        },
                         "path": {
-                            "type": "object",
+                            "default": {
+                                "type": "PathPrefix",
+                                "value": "/"
+                            },
+                            "description": "Path specifies a HTTP request path matcher. If this field is not\nspecified, a default prefix match on the \"/\" path is provided.",
                             "properties": {
                                 "type": {
-                                    "type": "string",
-                                    "enum": ["Exact", "PathPrefix", "RegularExpression"],
-                                    "default": "PathPrefix"
+                                    "default": "PathPrefix",
+                                    "description": "Type specifies how to match against the path Value.\n\n\nSupport: Core (Exact, PathPrefix)\n\n\nSupport: Implementation-specific (RegularExpression)",
+                                    "enum": [
+                                        "Exact",
+                                        "PathPrefix",
+                                        "RegularExpression"
+                                    ],
+                                    "type": "string"
                                 },
-                                "value": {"type": "string"}
+                                "value": {
+                                    "default": "/",
+                                    "description": "Value of the HTTP path to match against.",
+                                    "maxLength": 1024,
+                                    "type": "string"
+                                }
                             },
-                            "additionalProperties": False
+                            "type": "object",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "value must be an absolute path and start with '/' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? self.value.startsWith('/') : true"
+                                },
+                                {
+                                    "message": "must not contain '//' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('//') : true"
+                                },
+                                {
+                                    "message": "must not contain '/./' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('/./') : true"
+                                },
+                                {
+                                    "message": "must not contain '/../' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('/../') : true"
+                                },
+                                {
+                                    "message": "must not contain '%2f' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('%2f') : true"
+                                },
+                                {
+                                    "message": "must not contain '%2F' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('%2F') : true"
+                                },
+                                {
+                                    "message": "must not contain '#' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('#') : true"
+                                },
+                                {
+                                    "message": "must not end with '/..' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.endsWith('/..') : true"
+                                },
+                                {
+                                    "message": "must not end with '/.' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.endsWith('/.') : true"
+                                },
+                                {
+                                    "message": "type must be one of ['Exact', 'PathPrefix', 'RegularExpression']",
+                                    "rule": "self.type in ['Exact','PathPrefix'] || self.type == 'RegularExpression'"
+                                },
+                                {
+                                    "message": "must only contain valid characters (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) for types ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? self.value.matches(r\"\"\"^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$\"\"\") : true"
+                                }
+                            ]
                         },
-                        # Headers specifies HTTP request header matchers. Multiple match values are ANDed together, # noqa
-                        # meaning, a request must match all the specified headers to select the route. # noqa
-                        # gateway.networking.k8s.io/v1beta1.HTTPHeaderMatch
-                        # More info: https: // gateway-api.sigs.k8s.io/references/spec /
-                        "headers": {
-                            "type": "array",
+                        "queryParams": {
+                            "description": "QueryParams specifies HTTP query parameter matchers. Multiple match\nvalues are ANDed together, meaning, a request must match all the\nspecified query parameters to select the route.\n\n\nSupport: Extended",
                             "items": {
+                                "description": "HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP\nquery parameters.",
                                 "properties": {
+                                    "name": {
+                                        "description": "Name is the name of the HTTP query param to be matched. This must be an\nexact string match. (See\nhttps://tools.ietf.org/html/rfc7230#section-2.7.3).\n\n\nIf multiple entries specify equivalent query param names, only the first\nentry with an equivalent name MUST be considered for a match. Subsequent\nentries with an equivalent query param name MUST be ignored.\n\n\nIf a query param is repeated in an HTTP request, the behavior is\npurposely left undefined, since different data planes have different\ncapabilities. However, it is *recommended* that implementations should\nmatch against the first value of the param if the data plane supports it,\nas this behavior is expected in other load balancing contexts outside of\nthe Gateway API.\n\n\nUsers SHOULD NOT route traffic based on repeated query params to guard\nthemselves against potential differences in the implementations.",
+                                        "maxLength": 256,
+                                        "minLength": 1,
+                                        "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                        "type": "string"
+                                    },
                                     "type": {
-                                        "type": "string",
-                                        "enum": ["Exact", "RegularExpression"],
-                                        "default": "Exact"
+                                        "default": "Exact",
+                                        "description": "Type specifies how to match against the value of the query parameter.\n\n\nSupport: Extended (Exact)\n\n\nSupport: Implementation-specific (RegularExpression)\n\n\nSince RegularExpression QueryParamMatchType has Implementation-specific\nconformance, implementations can support POSIX, PCRE or any other\ndialects of regular expressions. Please read the implementation's\ndocumentation to determine the supported dialect.",
+                                        "enum": [
+                                            "Exact",
+                                            "RegularExpression"
+                                        ],
+                                        "type": "string"
                                     },
-                                    "name": {"type": "string"},
-                                    "value": {"type": "string"}
+                                    "value": {
+                                        "description": "Value is the value of HTTP query param to be matched.",
+                                        "maxLength": 1024,
+                                        "minLength": 1,
+                                        "type": "string"
+                                    }
                                 },
-                                "additionalProperties": False
+                                "required": [
+                                    "name",
+                                    "value"
+                                ],
+                                "type": "object"
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-list-map-keys": [
+                                "name"
+                            ],
+                            "x-kubernetes-list-type": "map"
+                        }
+                    },
+                    "type": "object"
+                },
+                "maxItems": 8,
+                "type": "array"
+            },
+            "sessionPersistence": {
+                "description": "SessionPersistence defines and configures session persistence\nfor the route rule.\n\n\nSupport: Extended\n\n\n",
+                "properties": {
+                    "absoluteTimeout": {
+                        "description": "AbsoluteTimeout defines the absolute timeout of the persistent\nsession. Once the AbsoluteTimeout duration has elapsed, the\nsession becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "cookieConfig": {
+                        "description": "CookieConfig provides configuration settings that are specific\nto cookie-based session persistence.\n\n\nSupport: Core",
+                        "properties": {
+                            "lifetimeType": {
+                                "default": "Session",
+                                "description": "LifetimeType specifies whether the cookie has a permanent or\nsession-based lifetime. A permanent cookie persists until its\nspecified expiry time, defined by the Expires or Max-Age cookie\nattributes, while a session cookie is deleted when the current\nsession ends.\n\n\nWhen set to \"Permanent\", AbsoluteTimeout indicates the\ncookie's lifetime via the Expires or Max-Age cookie attributes\nand is required.\n\n\nWhen set to \"Session\", AbsoluteTimeout indicates the\nabsolute lifetime of the cookie tracked by the gateway and\nis optional.\n\n\nSupport: Core for \"Session\" type\n\n\nSupport: Extended for \"Permanent\" type",
+                                "enum": [
+                                    "Permanent",
+                                    "Session"
+                                ],
+                                "type": "string"
                             }
                         },
-                        # QueryParams specifies HTTP query parameter matchers. Multiple match values are ANDed together, # noqa
-                        # meaning, a request must match all the specified query parameters to select the route. # noqa
-                        "queryParams": {
-                            "type": "array",
+                        "type": "object"
+                    },
+                    "idleTimeout": {
+                        "description": "IdleTimeout defines the idle timeout of the persistent session.\nOnce the session has been idle for more than the specified\nIdleTimeout duration, the session becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "sessionName": {
+                        "description": "SessionName defines the name of the persistent session token\nwhich may be reflected in the cookie or the header. Users\nshould avoid reusing session names to prevent unintended\nconsequences, such as rejection or unpredictable behavior.\n\n\nSupport: Implementation-specific",
+                        "maxLength": 128,
+                        "type": "string"
+                    },
+                    "type": {
+                        "default": "Cookie",
+                        "description": "Type defines the type of session persistence such as through\nthe use a header or cookie. Defaults to cookie based session\npersistence.\n\n\nSupport: Core for \"Cookie\" type\n\n\nSupport: Extended for \"Header\" type",
+                        "enum": [
+                            "Cookie",
+                            "Header"
+                        ],
+                        "type": "string"
+                    }
+                },
+                "type": "object",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "AbsoluteTimeout must be specified when cookie lifetimeType is Permanent",
+                        "rule": "!has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != 'Permanent' || has(self.absoluteTimeout)"
+                    }
+                ]
+            },
+            "timeouts": {
+                "description": "Timeouts defines the timeouts that can be configured for an HTTP request.\n\n\nSupport: Extended\n\n\n",
+                "properties": {
+                    "backendRequest": {
+                        "description": "BackendRequest specifies a timeout for an individual request from the gateway\nto a backend. This covers the time from when the request first starts being\nsent from the gateway to when the full response has been received from the backend.\n\n\nSetting a timeout to the zero duration (e.g. \"0s\") SHOULD disable the timeout\ncompletely. Implementations that cannot completely disable the timeout MUST\ninstead interpret the zero duration as the longest possible value to which\nthe timeout can be set.\n\n\nAn entire client HTTP transaction with a gateway, covered by the Request timeout,\nmay result in more than one call from the gateway to the destination backend,\nfor example, if automatic retries are supported.\n\n\nBecause the Request timeout encompasses the BackendRequest timeout, the value of\nBackendRequest must be <= the value of Request timeout.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "request": {
+                        "description": "Request specifies the maximum duration for a gateway to respond to an HTTP request.\nIf the gateway has not been able to respond before this deadline is met, the gateway\nMUST return a timeout error.\n\n\nFor example, setting the `rules.timeouts.request` field to the value `10s` in an\n`HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds\nto complete.\n\n\nSetting a timeout to the zero duration (e.g. \"0s\") SHOULD disable the timeout\ncompletely. Implementations that cannot completely disable the timeout MUST\ninstead interpret the zero duration as the longest possible value to which\nthe timeout can be set.\n\n\nThis timeout is intended to cover as close to the whole request-response transaction\nas possible although an implementation MAY choose to start the timeout after the entire\nrequest stream has been received instead of immediately after the transaction is\ninitiated by the client.\n\n\nWhen this field is unspecified, request timeout behavior is implementation-specific.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    }
+                },
+                "type": "object",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "backendRequest timeout cannot be longer than request timeout",
+                        "rule": "!(has(self.request) && has(self.backendRequest) && duration(self.request) != duration('0s') && duration(self.backendRequest) > duration(self.request))"
+                    }
+                ]
+            }
+        },
+        "type": "object",
+        "x-kubernetes-validations": [
+            {
+                "message": "RequestRedirect filter must not be used together with backendRefs",
+                "rule": "(has(self.backendRefs) && size(self.backendRefs) > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): true"
+            },
+            {
+                "message": "When using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.filters) && self.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == 'ReplacePrefixMatch' && has(f.requestRedirect.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            },
+            {
+                "message": "When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.filters) && self.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == 'ReplacePrefixMatch' && has(f.urlRewrite.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            },
+            {
+                "message": "Within backendRefs, when using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == 'ReplacePrefixMatch' && has(f.requestRedirect.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            },
+            {
+                "message": "Within backendRefs, When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == 'ReplacePrefixMatch' && has(f.urlRewrite.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            }
+        ]
+    },
+    "maxItems": 16,
+    "type": "array"
+}
+
+GRPC_RULES_SCHEMA = {
+    "description": "Rules are a list of GRPC matchers, filters and actions.",
+    "items": {
+        "description": "GRPCRouteRule defines the semantics for matching a gRPC request based on\nconditions (matches), processing it (filters), and forwarding the request to\nan API object (backendRefs).",
+        "properties": {
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent.\n\n\nFailure behavior here depends on how many BackendRefs are specified and\nhow many are invalid.\n\n\nIf *all* entries in BackendRefs are invalid, and there are also no filters\nspecified in this route rule, *all* traffic which matches this rule MUST\nreceive an `UNAVAILABLE` status.\n\n\nSee the GRPCBackendRef definition for the rules about what makes a single\nGRPCBackendRef invalid.\n\n\nWhen a GRPCBackendRef is invalid, `UNAVAILABLE` statuses MUST be returned for\nrequests that would have otherwise been routed to an invalid backend. If\nmultiple backends are specified, and some are invalid, the proportion of\nrequests that would otherwise have been routed to an invalid backend\nMUST receive an `UNAVAILABLE` status.\n\n\nFor example, if two backends are specified with equal weights, and one is\ninvalid, 50 percent of traffic MUST receive an `UNAVAILABLE` status.\nImplementations may choose how that 50 percent is determined.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Core",
+                "items": {
+                    "description": "GRPCBackendRef defines how a GRPCRoute forwards a gRPC request.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>",
+                    "properties": {
+                        "filters": {
+                            "description": "Filters defined at this level MUST be executed if and only if the\nrequest is being forwarded to the backend defined here.\n\n\nSupport: Implementation-specific (For broader support of filters, use the\nFilters field in GRPCRouteRule.)",
                             "items": {
+                                "description": "GRPCRouteFilter defines processing steps that must be completed during the\nrequest or response lifecycle. GRPCRouteFilters are meant as an extension\npoint to express processing that may be done in Gateway implementations. Some\nexamples include request or response modification, implementing\nauthentication strategies, rate-limiting, and traffic shaping. API\nguarantee/conformance is defined based on the type of the filter.",
                                 "properties": {
+                                    "extensionRef": {
+                                        "description": "ExtensionRef is an optional, implementation-specific extension to the\n\"filter\" behavior.  For example, resource \"myroutefilter\" in group\n\"networking.example.net\"). ExtensionRef MUST NOT be used for core and\nextended filters.\n\n\nSupport: Implementation-specific\n\n\nThis filter can be used multiple times within the same rule.",
+                                        "properties": {
+                                            "group": {
+                                                "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                                "maxLength": 253,
+                                                "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "kind": {
+                                                "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                                "maxLength": 63,
+                                                "minLength": 1,
+                                                "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                "type": "string"
+                                            },
+                                            "name": {
+                                                "description": "Name is the name of the referent.",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "group",
+                                            "kind",
+                                            "name"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "requestHeaderModifier": {
+                                        "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "requestMirror": {
+                                        "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "backendRef": {
+                                                "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                                "properties": {
+                                                    "group": {
+                                                        "default": "",
+                                                        "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                                        "maxLength": 253,
+                                                        "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                        "type": "string"
+                                                    },
+                                                    "kind": {
+                                                        "default": "Service",
+                                                        "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "name": {
+                                                        "description": "Name is the name of the referent.",
+                                                        "maxLength": 253,
+                                                        "minLength": 1,
+                                                        "type": "string"
+                                                    },
+                                                    "namespace": {
+                                                        "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "port": {
+                                                        "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                                        "format": "int32",
+                                                        "maximum": 65535,
+                                                        "minimum": 1,
+                                                        "type": "integer"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "name"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "Must have port for Service reference",
+                                                        "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                                    }
+                                                ]
+                                            }
+                                        },
+                                        "required": [
+                                            "backendRef"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "responseHeaderModifier": {
+                                        "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
                                     "type": {
-                                        "type": "string",
-                                        "enum": ["Exact", "RegularExpression"],
-                                        "default": "Exact"
+                                        "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations supporting GRPCRoute MUST support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` MUST be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\n",
+                                        "enum": [
+                                            "ResponseHeaderModifier",
+                                            "RequestHeaderModifier",
+                                            "RequestMirror",
+                                            "ExtensionRef"
+                                        ],
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "type"
+                                ],
+                                "type": "object",
+                                "x-kubernetes-validations": [
+                                    {
+                                        "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                                        "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                                        "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                                        "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                                        "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                                        "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
                                     },
-                                    "name": {"type": "string"},
-                                    "value": {"type": "string"}
+                                    {
+                                        "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                                        "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                                        "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                                        "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                                    }
+                                ]
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "RequestHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
                                 },
-                                "additionalProperties": False
-                            }
+                                {
+                                    "message": "ResponseHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                                }
+                            ]
                         },
-                        "method": {
-                            "type": "string",
-                            "enum": ["CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE"],  # noqa
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
                         }
                     },
-                    "additionalProperties": False
-                }
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "type": "array"
             },
-            # Filters define the filters that are applied to requests that match this rule.
-            # More info: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteFilter # noqa
             "filters": {
-                "type": "array",
+                "description": "Filters define the filters that are applied to requests that match\nthis rule.\n\n\nThe effects of ordering of multiple behaviors are currently unspecified.\nThis can change in the future based on feedback during the alpha stage.\n\n\nConformance-levels at this level are defined based on the type of filter:\n\n\n- ALL core filters MUST be supported by all implementations that support\n  GRPCRoute.\n- Implementers are encouraged to support extended filters.\n- Implementation-specific custom filters have no API guarantees across\n  implementations.\n\n\nSpecifying the same filter multiple times is not supported unless explicitly\nindicated in the filter.\n\n\nIf an implementation can not support a combination of filters, it must clearly\ndocument that limitation. In cases where incompatible or unsupported\nfilters are specified and cause the `Accepted` condition to be set to status\n`False`, implementations may use the `IncompatibleFilters` reason to specify\nthis configuration error.\n\n\nSupport: Core",
                 "items": {
-                    "properties": FILTER_SCHEMA,
-                    "additionalProperties": False
+                    "description": "GRPCRouteFilter defines processing steps that must be completed during the\nrequest or response lifecycle. GRPCRouteFilters are meant as an extension\npoint to express processing that may be done in Gateway implementations. Some\nexamples include request or response modification, implementing\nauthentication strategies, rate-limiting, and traffic shaping. API\nguarantee/conformance is defined based on the type of the filter.",
+                    "properties": {
+                        "extensionRef": {
+                            "description": "ExtensionRef is an optional, implementation-specific extension to the\n\"filter\" behavior.  For example, resource \"myroutefilter\" in group\n\"networking.example.net\"). ExtensionRef MUST NOT be used for core and\nextended filters.\n\n\nSupport: Implementation-specific\n\n\nThis filter can be used multiple times within the same rule.",
+                            "properties": {
+                                "group": {
+                                    "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                    "maxLength": 253,
+                                    "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "kind": {
+                                    "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                    "maxLength": 63,
+                                    "minLength": 1,
+                                    "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                    "type": "string"
+                                },
+                                "name": {
+                                    "description": "Name is the name of the referent.",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "group",
+                                "kind",
+                                "name"
+                            ],
+                            "type": "object"
+                        },
+                        "requestHeaderModifier": {
+                            "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "requestMirror": {
+                            "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                            "properties": {
+                                "backendRef": {
+                                    "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                    "properties": {
+                                        "group": {
+                                            "default": "",
+                                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                            "maxLength": 253,
+                                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                            "type": "string"
+                                        },
+                                        "kind": {
+                                            "default": "Service",
+                                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "name": {
+                                            "description": "Name is the name of the referent.",
+                                            "maxLength": 253,
+                                            "minLength": 1,
+                                            "type": "string"
+                                        },
+                                        "namespace": {
+                                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "port": {
+                                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                            "format": "int32",
+                                            "maximum": 65535,
+                                            "minimum": 1,
+                                            "type": "integer"
+                                        }
+                                    },
+                                    "required": [
+                                        "name"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "Must have port for Service reference",
+                                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                        }
+                                    ]
+                                }
+                            },
+                            "required": [
+                                "backendRef"
+                            ],
+                            "type": "object"
+                        },
+                        "responseHeaderModifier": {
+                            "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "type": {
+                            "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations supporting GRPCRoute MUST support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` MUST be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\n",
+                            "enum": [
+                                "ResponseHeaderModifier",
+                                "RequestHeaderModifier",
+                                "RequestMirror",
+                                "ExtensionRef"
+                            ],
+                            "type": "string"
+                        }
+                    },
+                    "required": [
+                        "type"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                            "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                            "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                            "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                            "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                            "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                            "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                            "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                            "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                        }
+                    ]
                 },
-            },
-            # BackendRefs defines the backend(s) where matching requests should be sent.
-            # More info: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPBackendRef # noqa
-            "backendRefs": {
+                "maxItems": 16,
                 "type": "array",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "RequestHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                    },
+                    {
+                        "message": "ResponseHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                    }
+                ]
+            },
+            "matches": {
+                "description": "Matches define conditions used for matching the rule against incoming\ngRPC requests. Each match is independent, i.e. this rule will be matched\nif **any** one of the matches is satisfied.\n\n\nFor example, take the following matches configuration:\n\n\n```\nmatches:\n- method:\n    service: foo.bar\n  headers:\n    values:\n      version: 2\n- method:\n    service: foo.bar.v2\n```\n\n\nFor a request to match against this rule, it MUST satisfy\nEITHER of the two conditions:\n\n\n- service of foo.bar AND contains the header `version: 2`\n- service of foo.bar.v2\n\n\nSee the documentation for GRPCRouteMatch on how to specify multiple\nmatch conditions to be ANDed together.\n\n\nIf no matches are specified, the implementation MUST match every gRPC request.\n\n\nProxy or Load Balancer routing configuration generated from GRPCRoutes\nMUST prioritize rules based on the following criteria, continuing on\nties. Merging MUST not be done between GRPCRoutes and HTTPRoutes.\nPrecedence MUST be given to the rule with the largest number of:\n\n\n* Characters in a matching non-wildcard hostname.\n* Characters in a matching hostname.\n* Characters in a matching service.\n* Characters in a matching method.\n* Header matches.\n\n\nIf ties still exist across multiple Routes, matching precedence MUST be\ndetermined in order of the following criteria, continuing on ties:\n\n\n* The oldest Route based on creation timestamp.\n* The Route appearing first in alphabetical order by\n  \"{namespace}/{name}\".\n\n\nIf ties still exist within the Route that has been given precedence,\nmatching precedence MUST be granted to the first matching rule meeting\nthe above criteria.",
                 "items": {
+                    "description": "GRPCRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\n\nFor example, the match below will match a gRPC request only if its service\nis `foo` AND it contains the `version: v1` header:\n\n\n```\nmatches:\n  - method:\n    type: Exact\n    service: \"foo\"\n    headers:\n  - name: \"version\"\n    value \"v1\"\n\n\n```",
                     "properties": {
-                        "filters": {
-                            "type": "array",
+                        "headers": {
+                            "description": "Headers specifies gRPC request header matchers. Multiple match values are\nANDed together, meaning, a request MUST match all the specified headers\nto select the route.",
                             "items": {
-                                "properties": FILTER_SCHEMA,
-                                "additionalProperties": False
+                                "description": "GRPCHeaderMatch describes how to select a gRPC route by matching gRPC request\nheaders.",
+                                "properties": {
+                                    "name": {
+                                        "description": "Name is the name of the gRPC Header to be matched.\n\n\nIf multiple entries specify equivalent header names, only the first\nentry with an equivalent name MUST be considered for a match. Subsequent\nentries with an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                        "maxLength": 256,
+                                        "minLength": 1,
+                                        "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                        "type": "string"
+                                    },
+                                    "type": {
+                                        "default": "Exact",
+                                        "description": "Type specifies how to match against the value of the header.",
+                                        "enum": [
+                                            "Exact",
+                                            "RegularExpression"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "value": {
+                                        "description": "Value is the value of the gRPC Header to be matched.",
+                                        "maxLength": 4096,
+                                        "minLength": 1,
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "name",
+                                    "value"
+                                ],
+                                "type": "object"
                             },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-list-map-keys": [
+                                "name"
+                            ],
+                            "x-kubernetes-list-type": "map"
                         },
-                        "group": {"type": "string"},
-                        "kind": {"type": "string"},
-                        "name": {"type": "string"},
-                        "namespace": {"type": "string"},
-                        "port": {"type": "integer"},
-                        "weight": {"type": "integer"}
+                        "method": {
+                            "description": "Method specifies a gRPC request service/method matcher. If this field is\nnot specified, all services and methods will match.",
+                            "properties": {
+                                "method": {
+                                    "description": "Value of the method to match against. If left empty or omitted, will\nmatch all services.\n\n\nAt least one of Service and Method MUST be a non-empty string.",
+                                    "maxLength": 1024,
+                                    "type": "string"
+                                },
+                                "service": {
+                                    "description": "Value of the service to match against. If left empty or omitted, will\nmatch any service.\n\n\nAt least one of Service and Method MUST be a non-empty string.",
+                                    "maxLength": 1024,
+                                    "type": "string"
+                                },
+                                "type": {
+                                    "default": "Exact",
+                                    "description": "Type specifies how to match against the service and/or method.\nSupport: Core (Exact with service and method specified)\n\n\nSupport: Implementation-specific (Exact with method specified but no service specified)\n\n\nSupport: Implementation-specific (RegularExpression)",
+                                    "enum": [
+                                        "Exact",
+                                        "RegularExpression"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "One or both of 'service' or 'method' must be specified",
+                                    "rule": "has(self.type) ? has(self.service) || has(self.method) : true"
+                                },
+                                {
+                                    "message": "service must only contain valid characters (matching ^(?i)\\.?[a-z_][a-z_0-9]*(\\.[a-z_][a-z_0-9]*)*$)",
+                                    "rule": "(!has(self.type) || self.type == 'Exact') && has(self.service) ? self.service.matches(r\"\"\"^(?i)\\.?[a-z_][a-z_0-9]*(\\.[a-z_][a-z_0-9]*)*$\"\"\"): true"
+                                },
+                                {
+                                    "message": "method must only contain valid characters (matching ^[A-Za-z_][A-Za-z_0-9]*$)",
+                                    "rule": "(!has(self.type) || self.type == 'Exact') && has(self.method) ? self.method.matches(r\"\"\"^[A-Za-z_][A-Za-z_0-9]*$\"\"\"): true"
+                                }
+                            ]
+                        }
                     },
-                    "required": ["name"],
-                    "additionalProperties": False
-                }
+                    "type": "object"
+                },
+                "maxItems": 8,
+                "type": "array"
             },
-            "timeouts": {
-                "request": {"type": "string"},
-                "backendRequest": {"type": "string"}
+            "sessionPersistence": {
+                "description": "SessionPersistence defines and configures session persistence\nfor the route rule.\n\n\nSupport: Extended\n\n\n",
+                "properties": {
+                    "absoluteTimeout": {
+                        "description": "AbsoluteTimeout defines the absolute timeout of the persistent\nsession. Once the AbsoluteTimeout duration has elapsed, the\nsession becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "cookieConfig": {
+                        "description": "CookieConfig provides configuration settings that are specific\nto cookie-based session persistence.\n\n\nSupport: Core",
+                        "properties": {
+                            "lifetimeType": {
+                                "default": "Session",
+                                "description": "LifetimeType specifies whether the cookie has a permanent or\nsession-based lifetime. A permanent cookie persists until its\nspecified expiry time, defined by the Expires or Max-Age cookie\nattributes, while a session cookie is deleted when the current\nsession ends.\n\n\nWhen set to \"Permanent\", AbsoluteTimeout indicates the\ncookie's lifetime via the Expires or Max-Age cookie attributes\nand is required.\n\n\nWhen set to \"Session\", AbsoluteTimeout indicates the\nabsolute lifetime of the cookie tracked by the gateway and\nis optional.\n\n\nSupport: Core for \"Session\" type\n\n\nSupport: Extended for \"Permanent\" type",
+                                "enum": [
+                                    "Permanent",
+                                    "Session"
+                                ],
+                                "type": "string"
+                            }
+                        },
+                        "type": "object"
+                    },
+                    "idleTimeout": {
+                        "description": "IdleTimeout defines the idle timeout of the persistent session.\nOnce the session has been idle for more than the specified\nIdleTimeout duration, the session becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "sessionName": {
+                        "description": "SessionName defines the name of the persistent session token\nwhich may be reflected in the cookie or the header. Users\nshould avoid reusing session names to prevent unintended\nconsequences, such as rejection or unpredictable behavior.\n\n\nSupport: Implementation-specific",
+                        "maxLength": 128,
+                        "type": "string"
+                    },
+                    "type": {
+                        "default": "Cookie",
+                        "description": "Type defines the type of session persistence such as through\nthe use a header or cookie. Defaults to cookie based session\npersistence.\n\n\nSupport: Core for \"Cookie\" type\n\n\nSupport: Extended for \"Header\" type",
+                        "enum": [
+                            "Cookie",
+                            "Header"
+                        ],
+                        "type": "string"
+                    }
+                },
+                "type": "object",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "AbsoluteTimeout must be specified when cookie lifetimeType is Permanent",
+                        "rule": "!has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != 'Permanent' || has(self.absoluteTimeout)"
+                    }
+                ]
             }
         },
-        "required": ["backendRefs"],
-        "additionalProperties": False
-    }
+        "type": "object"
+    },
+    "maxItems": 16,
+    "type": "array",
 }
 
 SCHEMA = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "properties": {
-        "canary": RULES_SCHEMA,
-        "stable": RULES_SCHEMA,
-    },
-    "required": ["canary", "stable"],
-    "additionalProperties": False
+  "anyOf": [
+    TCP_RULES_SCHEMA,
+    UDP_RULES_SCHEMA,
+    TLS_RULES_SCHEMA,
+    HTTP_RULES_SCHEMA,
+    GRPC_RULES_SCHEMA,
+  ],
+  "$schema": "http://json-schema.org/schema#"
 }
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index d075d33e2..823fcba19 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -250,7 +250,6 @@
 
 # names which apps cannot reserve for routing
 DRYCC_RESERVED_NAMES = os.environ.get('RESERVED_NAMES', '').replace(' ', '').split(',')
-DRYCC_RESERVED_NAMES.append("canary")
 
 # the k8s namespace in which the controller and workflow were installed.
 WORKFLOW_NAMESPACE = os.environ.get('WORKFLOW_NAMESPACE', 'drycc')
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index 0301a8654..fc127fe05 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -203,25 +203,12 @@ def domain_changed_handle(
 @receiver(signal=[post_save, post_delete], sender=AppSettings)
 def appsettings_changed_handle(
         sender, instance: AppSettings, created=False, update_fields=None, **kwargs):
-    prev_settings, action, canaries = instance.diff_canaries()
-    if prev_settings is not None:
-        release = instance.app.release_set.filter(failed=False).latest()
-        if release.canary:
-            if action == "append":
-                instance.app.deploy(release)
-            elif action == "remove":
-                instance.app.cleanup_old()
-        for procfile_type in canaries:
-            canary = (action == "append")
-            service = instance.app.service_set.filter(procfile_type=procfile_type).first()
-            if service is not None and service.canary != canary:
-                service.canary = canary
-                service.save()
-        if prev_settings.routable != instance.routable:
-            for route in instance.app.route_set.all():
-                if route.routable != instance.routable:
-                    route.routable = instance.routable
-                    route.save()
+    prev_settings = instance.previous()
+    if prev_settings and prev_settings.routable != instance.routable:
+        for route in instance.app.route_set.all():
+            if route.routable != instance.routable:
+                route.routable = instance.routable
+                route.save()
 
 
 @receiver(post_save, sender=Config)
diff --git a/rootfs/api/tests/rules/grpc_rules.json b/rootfs/api/tests/rules/grpc_rules.json
index 925787558..6a882a0a3 100644
--- a/rootfs/api/tests/rules/grpc_rules.json
+++ b/rootfs/api/tests/rules/grpc_rules.json
@@ -1,56 +1,33 @@
-{
-	"stable": [{
-			"matches": [{
+[
+	{
+		"matches": [
+			{
 				"method": {
 					"service": "com.example.User",
 					"method": "Login"
 				}
-			}],
-			"backendRefs": [{
+			}
+		],
+		"backendRefs": [
+			{
 				"name": "foo-svc",
 				"port": 50051
-			}]
-		},
-		{
-			"matches": [{
+			}
+		]
+	},
+	{
+		"matches": [
+			{
 				"method": {
 					"service": "grpc.reflection.v1.ServerReflection"
 				}
-			}],
-			"backendRefs": [{
+			}
+		],
+		"backendRefs": [
+			{
 				"name": "foo-svc",
 				"port": 50051
-			}]
-		}
-	],
-	"canary": [{
-			"matches": [{
-				"method": {
-					"service": "com.example.User",
-					"method": "Login"
-				}
-			}],
-			"backendRefs": [{
-				"name": "foo-svc",
-				"port": 50051
-			}, {
-				"name": "foo-svc-canary",
-				"port": 50051
-			}]
-		},
-		{
-			"matches": [{
-				"method": {
-					"service": "grpc.reflection.v1.ServerReflection"
-				}
-			}],
-			"backendRefs": [{
-				"name": "foo-svc",
-				"port": 50051
-			}, {
-				"name": "foo-svc-canary",
-				"port": 50051
-			}]
-		}
-	]
-}
\ No newline at end of file
+			}
+		]
+	}
+]
\ No newline at end of file
diff --git a/rootfs/api/tests/rules/http_rules.json b/rootfs/api/tests/rules/http_rules.json
index 5c4371350..a7cf7b6b0 100644
--- a/rootfs/api/tests/rules/http_rules.json
+++ b/rootfs/api/tests/rules/http_rules.json
@@ -1,73 +1,43 @@
-{
-	"stable": [{
-		"matches": [{
-			"path": {
-				"type": "PathPrefix",
-				"value": "/add-multiple-response-headers"
+[
+	{
+		"matches": [
+			{
+				"path": {
+					"type": "PathPrefix",
+					"value": "/add-multiple-response-headers"
+				}
+			}
+		],
+		"filters": [
+			{
+				"type": "ResponseHeaderModifier",
+				"responseHeaderModifier": {
+					"add": [
+						{
+							"name": "X-Header-Add-1",
+							"value": "header-add-1"
+						},
+						{
+							"name": "X-Header-Add-2",
+							"value": "header-add-2"
+						},
+						{
+							"name": "X-Header-Add-3",
+							"value": "header-add-3"
+						}
+					]
+				}
 			}
-		}],
-		"filters": [{
-			"type": "ResponseHeaderModifier",
-			"responseHeaderModifier": {
-				"add": [{
-						"name": "X-Header-Add-1",
-						"value": "header-add-1"
-					},
-					{
-						"name": "X-Header-Add-2",
-						"value": "header-add-2"
-					},
-					{
-						"name": "X-Header-Add-3",
-						"value": "header-add-3"
-					}
-				]
+		],
+		"backendRefs": [
+			{
+				"name": "echo",
+				"port": 8080
 			}
-		}],
-		"backendRefs": [{
-			"name": "echo",
-			"port": 8080
-		}],
+		],
 		"timeouts": {
 			"backendRequest": "60s",
 			"request": "60s"
 		}
-	}],
-	"canary": [{
-		"matches": [{
-			"path": {
-				"type": "PathPrefix",
-				"value": "/add-multiple-response-headers"
-			}
-		}],
-		"filters": [{
-			"type": "ResponseHeaderModifier",
-			"responseHeaderModifier": {
-				"add": [{
-						"name": "X-Header-Add-1",
-						"value": "header-add-1"
-					},
-					{
-						"name": "X-Header-Add-2",
-						"value": "header-add-2"
-					},
-					{
-						"name": "X-Header-Add-3",
-						"value": "header-add-3"
-					}
-				]
-			}
-		}],
-		"backendRefs": [{
-				"name": "echo",
-				"port": 8080,
-				"weight": 10
-			},
-			{
-				"name": "echo-canary",
-				"port": 8080,
-				"weight": 90
-			}
-		]
-	}]
-}
\ No newline at end of file
+	}
+]
\ No newline at end of file
diff --git a/rootfs/api/tests/rules/tcp_rules.json b/rootfs/api/tests/rules/tcp_rules.json
index d13cab8cc..0ca5ea93f 100644
--- a/rootfs/api/tests/rules/tcp_rules.json
+++ b/rootfs/api/tests/rules/tcp_rules.json
@@ -1,21 +1,11 @@
-{
-	"stable": [{
-		"backendRefs": [{
-			"name": "echo",
-			"port": 8080
-		}]
-	}],
-	"canary": [{
-		"backendRefs": [{
+[
+	{
+		"backendRefs": [
+			{
 				"name": "echo",
 				"port": 8080,
 				"weight": 10
-			},
-			{
-				"name": "echo-canary",
-				"port": 8080,
-				"weight": 90
 			}
 		]
-	}]
-}
\ No newline at end of file
+	}
+]
\ No newline at end of file
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
index 08980dae2..75cf0f2d3 100644
--- a/rootfs/api/tests/test_app_settings.py
+++ b/rootfs/api/tests/test_app_settings.py
@@ -171,41 +171,3 @@ def test_settings_labels(self, mock_requests):
             labels
         )
         self.assertEqual(response.status_code, 422, response.data)
-
-    def test_canaries(self, mock_requests):
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
-        expect = ["web", "task", "new", "apps"]
-        for index, scale_type in enumerate(expect):
-            response = self.client.post(
-                '/v2/apps/{}/services'.format(app_id),
-                {
-                    'port': 5000 + index,
-                    'protocol': 'TCP',
-                    'target_port': 5000 + index,
-                    'procfile_type': scale_type
-                }
-            )
-            self.assertEqual(response.status_code, 201, response.data)
-        self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': expect[:2]}
-        )
-        response = self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': expect[2:]}
-        )
-        expect = ["web", "task", "new", "apps"]
-        self.assertEqual(
-            response.json()["canaries"],
-            expect,
-            response.data
-        )
-
-        self.assertEqual(len(app.service_set.all()), 4)
-        response = self.client.delete(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': ["new", "apps"]}
-        )
-        app_settings = app.appsettings_set.latest()
-        self.assertEqual(app_settings.canaries, ["web", "task"])
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index baa7ba388..898cbf045 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -253,7 +253,6 @@ def test_build_forgotten_procfile(self, mock_requests):
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
@@ -798,13 +797,13 @@ def test_dryccfile_format(self, mock_requests):
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
             body['dryccfile']['deploy'] = {
-                "web-canary": {
+                "web-new": {
                     'image': "127.0.0.1/cat/cat"
                 }
             }
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400, response.data)
-            del body['dryccfile']['deploy']['web-canary']
+            self.assertEqual(response.status_code, 201, response.data)
+            del body['dryccfile']['deploy']['web-new']
             body['dryccfile']['deploy'] = {
                 "web": {
                     'image': "127.0.0.1/cat/cat"
diff --git a/rootfs/api/tests/test_canary.py b/rootfs/api/tests/test_canary.py
deleted file mode 100644
index fcb55cc53..000000000
--- a/rootfs/api/tests/test_canary.py
+++ /dev/null
@@ -1,90 +0,0 @@
-import json
-from django.contrib.auth import get_user_model
-from django.core.cache import cache
-
-from api.models.app import App
-from api.tests import adapter, DryccTransactionTestCase
-import requests_mock
-
-User = get_user_model()
-
-
-@requests_mock.Mocker(real_http=True, adapter=adapter)
-class CanaryTest(DryccTransactionTestCase):
-
-    """Tests push notification from build system"""
-
-    fixtures = ['tests.json']
-
-    def setUp(self):
-        self.user = User.objects.get(username='autotest')
-        self.token = self.get_or_create_token(self.user)
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-    def tearDown(self):
-        # make sure every test has a clean slate for k8s mocking
-        cache.clear()
-
-    def pre_data(self):
-        app_id = self.create_app()
-        # check that updating config rolls a new release
-        url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
-        # check that updating the build rolls a new release
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:v1', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['image'], body['image'])
-        # create v2 release
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:v2', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['image'], body['image'])
-        # get deployments
-        app = App.objects.get(id=app_id)
-        response = app.scheduler().deployments.get(app_id)
-        self.assertEqual(len(response.json()["items"]), 1)
-        # add canary
-        self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': ["web"]}
-        )
-        # add v3 release
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:v3', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['image'], body['image'])
-        return app
-
-    def test_release(self, mock_requests):
-        app = self.pre_data()
-        # get deployments
-        response = app.scheduler().deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 2)
-        response = self.client.post(f'/v2/apps/{app.id}/canary/release/')
-        self.assertEqual(response.status_code, 201)
-        response = app.scheduler().deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 1)
-        self.assertEqual(
-            response.json()["items"][0]["spec"]["template"]["spec"]["containers"][0]["image"],
-            'autotest/example:v3',
-        )
-
-    def test_rollback(self, mock_requests):
-        app = self.pre_data()
-        response = app.scheduler().deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 2)
-        response = self.client.post(f'/v2/apps/{app.id}/canary/rollback/')
-        self.assertEqual(response.status_code, 201)
-        response = app.scheduler().deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 1)
-        self.assertEqual(
-            response.json()["items"][0]["spec"]["template"]["spec"]["containers"][0]["image"],
-            'autotest/example:v2',
-        )
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index c96aec747..340774948 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -608,67 +608,36 @@ def test_rule_get(self):
         response = self.client.get(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
         )
-        expect = {
-            'canary': [{
-                'backendRefs': [{
-                    'kind': 'Service',
-                    'name': '%s-%s' % (app_id, procfile_type),
-                    'port': 5000,
-                    'weight': 100
-                }, {
-                    'kind': 'Service',
-                    'name': '%s-%s-canary' % (app_id, procfile_type),
-                    'port': 5000,
-                    'weight': 0
-                }]
-            }],
-            'stable': [{
-                'backendRefs': [{
-                    'kind': 'Service',
-                    'name': '%s-%s' % (app_id, procfile_type),
-                    'port': 5000,
-                    'weight': 100
-                }]
+        expect = [{
+            'backendRefs': [{
+                'kind': 'Service',
+                'name': '%s-%s' % (app_id, procfile_type),
+                'port': 5000,
+                'weight': 100
             }]
-        }
-        self.assertEqual(response.data["stable"], expect["stable"])
-        self.assertEqual(response.data["canary"], expect["canary"])
+        }]
+        self.assertEqual(response.data, expect)
 
     def test_schemas(self):
         for rule in os.listdir("{}/rules/".format(TEST_ROOT)):
             with open("{}/rules/{}".format(TEST_ROOT, rule)) as f:
                 data = f.read()
                 try:
-                    validate_json(data, RULES_SCHEMA)
+                    validate_json(json.loads(data), RULES_SCHEMA)
                 except Exception as e:
                     raise self.failureException("validate %s rule error: %s" % (rule, str(e)))
 
     def test_rule_set(self):
         app_id = self.create_app()
         procfile_type, _, route_name = self.create_route(app_id)
-        expect = {
-            "canary": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }, {
-                    "kind": "Service",
-                    "name": "%s-%s-canary" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 0
-                }]
-            }],
-            "stable": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }]
+        expect = [{
+            "backendRefs": [{
+                "kind": "Service",
+                "name": "%s-%s" % (app_id, procfile_type),
+                "port": 5000,
+                "weight": 100
             }]
-        }
+        }]
         response = self.client.put(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
             json.dumps(expect),
@@ -678,29 +647,14 @@ def test_rule_set(self):
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
         )
         self.assertEqual(json.dumps(response.data), json.dumps(expect))
-        expect = {
-            "canary": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s-noexits" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }, {
-                    "kind": "Service",
-                    "name": "%s-%s-noexits-canary" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 0
-                }]
-            }],
-            "stable": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s-noexits" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }]
+        expect = [{
+            "backendRefs": [{
+                "kind": "Service",
+                "name": "%s-%s-noexits" % (app_id, procfile_type),
+                "port": 5000,
+                "weight": 100
             }]
-        }
+        }]
         response = self.client.put(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
             json.dumps(expect),
diff --git a/rootfs/api/tests/test_procfile.py b/rootfs/api/tests/test_procfile.py
index 2748b2181..b8210ecd5 100644
--- a/rootfs/api/tests/test_procfile.py
+++ b/rootfs/api/tests/test_procfile.py
@@ -19,14 +19,14 @@ def assertException(self, method, exception=Exception, *args, **kwargs):
                 f"Did not throw the expected exception, args: {args}, kwargs: {kwargs}")
 
     def test_procfile_type_error(self):
-        for procfile_type in ["w", "we", "-a", "we-canary", "w" * 64]:
+        for procfile_type in ["w", "we", "-a", "we-new-", "w" * 64]:
             self.assertException(
                 validate_procfile_type, serializers.ValidationError, procfile_type)
 
     def test_procfile_type_ok(self):
         self.assertEqual(validate_procfile_type("web"), "web")
         self.assertEqual(validate_procfile_type("w" * 63), "w" * 63)
-        self.assertEqual(validate_procfile_type("web-canary-be"), "web-canary-be")
+        self.assertEqual(validate_procfile_type("web-new-be"), "web-new-be")
 
     def test_dryccfile_procfile_type(self):
         dryccfile_1 = {
@@ -56,15 +56,15 @@ def test_dryccfile_procfile_type(self):
         self.assertException(
                 validate_dryccfile, serializers.ValidationError, dryccfile_1)
         del dryccfile_1["deploy"]["w"]
-        dryccfile_1["deploy"]["w-canary"] = {"args": ["python", "-m", "http.server", "5000"]}
+        dryccfile_1["deploy"]["w-new-"] = {"args": ["python", "-m", "http.server", "5000"]}
         self.assertException(
                 validate_dryccfile, serializers.ValidationError, dryccfile_1)
 
     def test_staticmethod_procfile_type(self):
         s_validate_procfile_type = ServiceSerializer().validate_procfile_type
-        for procfile_type in ["w", "we", "-a", "we-canary", "w" * 64]:
+        for procfile_type in ["w", "we", "-a", "we-new-", "w" * 64]:
             self.assertException(
                 s_validate_procfile_type, serializers.ValidationError, procfile_type)
         self.assertEqual(s_validate_procfile_type("web"), "web")
         self.assertEqual(s_validate_procfile_type("w" * 63), "w" * 63)
-        self.assertEqual(s_validate_procfile_type("web-canary-be"), "web-canary-be")
+        self.assertEqual(s_validate_procfile_type("web-new-be"), "web-new-be")
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 009ebf39b..32702f771 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -188,7 +188,7 @@ def test_response_data(self, mock_requests):
         response = self.client.get(url)
         for key in response.data.keys():
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'build', 'config',
-                                'summary', 'canary', 'version', 'state', 'failed', 'exception'])
+                                'summary', 'version', 'state', 'failed', 'exception'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
@@ -220,7 +220,7 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         # create another release with a different build
         url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:canary', 'stack': 'container'}
+        body = {'image': 'autotest/example:latest', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # rollback and check to see that a 5th release was created
diff --git a/rootfs/api/tests/test_service.py b/rootfs/api/tests/test_service.py
index 39e21fe3a..144d1fb65 100644
--- a/rootfs/api/tests/test_service.py
+++ b/rootfs/api/tests/test_service.py
@@ -2,7 +2,6 @@
 from django.core.cache import cache
 from django.conf import settings
 
-from api.models.app import App
 from api.tests import DryccTransactionTestCase
 from api.tests.test_gateway import RouteTest
 
@@ -164,25 +163,3 @@ def test_service_basic_ops(self):
             {'procfile_type': 'test', "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 404, response.data)
-
-    def test_app_settings_change_canaries(self):
-        procfile_type, app_id, _, _, _, _ = self.test_route.test_route_attach()
-        # Add canaries
-        response = self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': [procfile_type]}
-        )
-        self.assertEqual(response.status_code, 201, response.data)
-        app = App.objects.get(id=app_id)
-        response = app.scheduler().svc.get(app_id)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.json()["items"]), 2)
-        # Remove canaries to false
-        response = self.client.delete(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': [procfile_type]}
-        )
-        self.assertEqual(response.status_code, 204, response.data)
-        response = app.scheduler().svc.get(app_id)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.json()["items"]), 1)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index ed84169a5..9eb09bab1 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -79,15 +79,7 @@
     # application settings
     re_path(
         r"^apps/(?P<id>{})/settings/?$".format(settings.APP_URL_REGEX),
-        views.AppSettingsViewSet.as_view(
-            {'get': 'retrieve', 'post': 'create', 'delete': 'remove'})),
-    # application canary
-    re_path(
-        r"^apps/(?P<id>{})/canary/release/?$".format(settings.APP_URL_REGEX),
-        views.CanaryViewSet.as_view({'post': 'release'})),
-    re_path(
-        r"^apps/(?P<id>{})/canary/rollback/?$".format(settings.APP_URL_REGEX),
-        views.CanaryViewSet.as_view({'post': 'rollback'})),
+        views.AppSettingsViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
     # application TLS settings
     re_path(
         r"^apps/(?P<id>{})/tls/?$".format(settings.APP_URL_REGEX),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index da1f93bb2..38f8265da 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -400,8 +400,7 @@ def post_save(self, config):
             raise DryccException('There is an executing pipeline, please wait')
         try:
             release = latest_release.new(
-                self.request.user, config=config, build=latest_release.build,
-                canary=latest_release.canary)
+                self.request.user, config=config, build=latest_release.build)
             if release.build is not None:
                 procfile_types = set()
                 for field, diff in config.diff().items():
@@ -473,42 +472,6 @@ class AppSettingsViewSet(AppResourceViewSet):
     model = models.appsettings.AppSettings
     serializer_class = serializers.AppSettingsSerializer
 
-    def remove(self, request, *args, **kwargs):
-        app = self.get_app()
-        app_settings = self.get_object()
-        new_canaries = []
-        remove_canaries = request.data["canaries"]
-        for procfile_tyle in app_settings.canaries:
-            if procfile_tyle not in remove_canaries:
-                new_canaries.append(procfile_tyle)
-        new_app_settings = self.model(app=app, owner=app.owner, canaries=new_canaries)
-        new_app_settings.summary = f"{app.owner} remove canaries {','.join(remove_canaries)}"
-        new_app_settings.save(ignore_update_field=["canaries"])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-class CanaryViewSet(AppResourceViewSet):
-    model = models.appsettings.AppSettings
-    serializer_class = serializers.AppSettingsSerializer
-
-    def _clean_canaries(self):
-        app = self.get_app()
-        app_settings = self.model(app=app, owner=app.owner, canaries=[])
-        app_settings.summary = "canary release"
-        app_settings.save(ignore_update_field=["canaries"])
-
-    def release(self, request, *args, **kwargs):
-        self._clean_canaries()
-        latest_release = self.get_app().release_set.filter(failed=False).latest()
-        data = {'version': latest_release.rollback(request.user, latest_release.version).version}
-        return Response(data, status=status.HTTP_201_CREATED)
-
-    def rollback(self, request, *args, **kwargs):
-        self._clean_canaries()
-        release = self.get_app().release_set.filter(failed=False, canary=False).latest()
-        data = {'version': release.rollback(request.user, release.version).version}
-        return Response(data, status=status.HTTP_201_CREATED)
-
 
 class DomainViewSet(AppResourceViewSet):
     """A viewset for interacting with Domain objects."""
@@ -562,7 +525,6 @@ def create_or_update(self, request, **kwargs):
             service = self.model(owner=app.owner, app=app, procfile_type=procfile_type)
             http_status = status.HTTP_201_CREATED
         service.add_port(port, protocol, target_port)
-        service.canary = procfile_type in app.appsettings_set.latest().canaries
         service.save()
         return Response(status=http_status)
 
@@ -1037,7 +999,7 @@ def set(self, request, *args, **kwargs):
         rules = request.data
         if isinstance(rules, str):
             rules = json.loads(rules)
-        rules = self.get_serializer().validate_rules(rules)
+        rules = self.get_serializer(route, many=False).validate_rules(rules)
         route.rules = rules
         ok, msg = route.check_rules()
         if not ok:
@@ -1067,7 +1029,7 @@ def create(self, request, *args, **kwargs):
             procfile_type=procfile_type,
         )
         route.rules = route.default_rules
-        if not route.current_rules[0]["backendRefs"]:
+        if route.rules and not route.rules[0]["backendRefs"]:
             return Response(status=status.HTTP_400_BAD_REQUEST, data={
                 "detail": "this route does not match services. please add service first."
             })
diff --git a/rootfs/scheduler/tests/test_gateway.py b/rootfs/scheduler/tests/test_gateway.py
index 72d00c2cf..6d6ec5491 100644
--- a/rootfs/scheduler/tests/test_gateway.py
+++ b/rootfs/scheduler/tests/test_gateway.py
@@ -89,11 +89,6 @@ def test_patch_http_route(self):
                         "name": "tcp-echo-v1",
                         "port": 9000,
                         "weight": 10,
-                    },
-                    {
-                        "name": "tcp-echo-v1-canary",
-                        "port": 9000,
-                        "weight": 90,
                     }
                 ]
             }
@@ -143,11 +138,6 @@ def test_patch_tcp_route(self):
                         "name": "tcp-echo-v1",
                         "port": 9000,
                         "weight": 10,
-                    },
-                    {
-                        "name": "tcp-echo-v1-canary",
-                        "port": 9000,
-                        "weight": 90,
                     }
                 ]
             }
@@ -197,11 +187,6 @@ def test_patch_udp_route(self):
                         "name": "udp-echo-v1",
                         "port": 9000,
                         "weight": 10,
-                    },
-                    {
-                        "name": "udp-echo-v1-canary",
-                        "port": 9000,
-                        "weight": 90,
                     }
                 ]
             }

From c6f785a8d271ef3c39fd93e1e68187ef481b972c Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 9 Jul 2024 15:29:46 +0800
Subject: [PATCH 033/200] chore(migrate): use helm hook replace cluster lock

---
 .../templates/controller-api-deployment.yaml  | 10 -----
 .../templates/controller-job-upgrade.yaml     |  4 +-
 .../api/management/commands/cluster_lock.py   | 42 -------------------
 3 files changed, 2 insertions(+), 54 deletions(-)
 delete mode 100644 rootfs/api/management/commands/cluster_lock.py

diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/controller-api-deployment.yaml
index 2c87eee85..7f61b98cb 100644
--- a/charts/controller/templates/controller-api-deployment.yaml
+++ b/charts/controller/templates/controller-api-deployment.yaml
@@ -38,16 +38,6 @@ spec:
         - -a
         - $(DRYCC_REDIS_ADDRS)
         {{- include "controller.envs" . | indent 8 }}
-      - name: drycc-controller-waitting
-        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-        imagePullPolicy: {{.Values.imagePullPolicy}}
-        args:
-        - /usr/bin/env
-        - bash
-        - -ec
-        - |
-          python -u /workspace/manage.py cluster_lock waitting
-        {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index e37d59a16..11663eee9 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -6,6 +6,8 @@ metadata:
     heritage: drycc
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
+    helm.sh/hook: post-install,pre-upgrade,pre-rollback
+    helm.sh/hook-delete-policy: hook-succeeded
 spec:
   template:
     metadata:
@@ -37,11 +39,9 @@ spec:
         - bash
         - -ec
         - |
-          python -u /workspace/manage.py cluster_lock lock
           python -u /workspace/manage.py migrate --noinput
           python -u /workspace/manage.py loaddata /etc/controller/limit-specs.json
           python -u /workspace/manage.py loaddata /etc/controller/limit-plans.json
-          python -u /workspace/manage.py cluster_lock unlock
         {{- end }}
         {{- include "controller.envs" . | indent 8 }}
         volumeMounts:
diff --git a/rootfs/api/management/commands/cluster_lock.py b/rootfs/api/management/commands/cluster_lock.py
deleted file mode 100644
index cd0560846..000000000
--- a/rootfs/api/management/commands/cluster_lock.py
+++ /dev/null
@@ -1,42 +0,0 @@
-import time
-from django.core.management.base import BaseCommand
-from django.core.cache import cache
-from django.conf import settings
-
-lock_key = "drycc:controller:version"
-waitting_init_msg = "version inconsistency %s!=%s, waiting for initialization to complete..."
-
-
-class Command(BaseCommand):
-    """Management command for push data to manager"""
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            "args",
-            metavar="action",
-            nargs="+",
-            choices=["lock", "unlock", "waitting"],
-            help="an action that needs to be executed.",
-        )
-
-    def lock(self):
-        cache.delete(lock_key)
-        print("lock completed!")
-
-    def unlock(self):
-        cache.set(lock_key, settings.VERSION, timeout=None)
-        print("unlock completed!")
-
-    def waitting(self):
-        while True:
-            version = cache.get(lock_key, None)
-            if version != settings.VERSION:
-                print(waitting_init_msg % (version, settings.VERSION))
-            else:
-                break
-            time.sleep(10)
-        print("waiting completed!")
-
-    def handle(self, *args, **options):
-        action = args[0]
-        getattr(self, action)()

From 180ad9b99dad3a1dd24466239323669c83dec41c Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 9 Jul 2024 16:30:54 +0800
Subject: [PATCH 034/200] feat(controller): add fetch ptypes and events

---
 .../templates/controller-clusterrole.yaml     |   2 +-
 rootfs/api/models/app.py                      | 101 +++++++++++++++
 rootfs/api/serializers/__init__.py            |  21 ++++
 rootfs/api/tasks.py                           |  17 +++
 rootfs/api/tests/test_build.py                |  10 +-
 rootfs/api/tests/test_config.py               |   2 +-
 rootfs/api/tests/test_event.py                |  77 ++++++++++++
 rootfs/api/tests/test_pods.py                 |  95 +++++---------
 rootfs/api/tests/test_ptypes.py               | 118 ++++++++++++++++++
 rootfs/api/urls.py                            |  31 +++--
 rootfs/api/views.py                           |  87 ++++++++++---
 rootfs/scheduler/mock.py                      |   7 +-
 rootfs/scheduler/resources/deployment.py      |  15 +--
 rootfs/scheduler/resources/events.py          |  28 +++--
 rootfs/scheduler/resources/namespace.py       |   8 --
 rootfs/scheduler/resources/pod.py             |  15 ++-
 rootfs/scheduler/tests/test_deployments.py    |  14 +--
 rootfs/scheduler/tests/test_events.py         |  38 ++++++
 18 files changed, 548 insertions(+), 138 deletions(-)
 create mode 100644 rootfs/api/tests/test_event.py
 create mode 100644 rootfs/api/tests/test_ptypes.py
 create mode 100644 rootfs/scheduler/tests/test_events.py

diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index 5835b4d39..34f521743 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -16,7 +16,7 @@ rules:
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "list"]
-- apiGroups: [""]
+- apiGroups: ["events.k8s.io"]
   resources: ["events"]
   verbs: ["list", "create"]
 - apiGroups: [""]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 576df08ba..627b83d55 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -427,6 +427,107 @@ def list_pods(self, *args, **kwargs):
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
 
+    def delete_pod(self, **kwargs):
+        """Used to list basic information about pods running for a given application"""
+        pod_name = kwargs.get('pod_name')
+        try:
+            # make sure the pod is manageed by drycc
+            pod = self.scheduler().pod.get(self.id, pod_name).json()
+            if pod['metadata']['labels'].get("heritage") == "drycc":
+                self.scheduler().pod.delete(self.id, pod_name)
+        except KubeHTTPException as e:
+            # Sometimes k8s will manage to remove the pod from under us
+            if e.response.status_code != 404:
+                raise e
+
+    def describe_deployment(self, deployment_name):
+        result = []
+        try:
+            deployment = self.scheduler().deployment.get(self.id, deployment_name).json()
+            for container in deployment["spec"]["template"]['spec']["containers"]:
+                limits = container.get("resources", {}).get("limits", {})
+                result.append({
+                    "container": container["name"],
+                    "image": container["image"],
+                    "command": container.get("command", []),
+                    "args": container.get("args", []),
+                    "liveness_probe": container.get("livenessProbe", {}),
+                    "readiness_probe": container.get("readinessProbe", {}),
+                    "limits": limits,
+                    "volume_mounts": container.get("volumeMounts", []),
+                })
+        except KubeHTTPException as e:
+            if e.response.status_code != 404:
+                raise e
+        return result
+
+    def list_deployments(self, *args, **kwargs):
+        """Used to list basic information about deployments running for a given application"""
+        try:
+            labels = self._scheduler_filter(**kwargs)
+            # in case a singular deployment is requested
+            if 'name' in kwargs:
+                deployments = [self.scheduler().deployment.get(self.id, kwargs['name']).json()]
+            else:
+                deployments = self.scheduler().deployment.get(self.id, labels=labels).json()['items']  # noqa
+                if not deployments:
+                    deployments = []
+            data = []
+            for p in deployments:
+                labels = p['spec']['template']['metadata']['labels']
+                if p['metadata']['creationTimestamp']:
+                    started = p['metadata']['creationTimestamp']
+                else:
+                    started = str(
+                        datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT))
+                item = {
+                    'name': labels['type'],
+                    'release': labels['version'],
+                    'ready': "%s/%s" % (
+                        p["status"]["readyReplicas"],
+                        p["status"]["replicas"],
+                    ),
+                    'up_to_date': p["status"]["updatedReplicas"],
+                    'available_replicas': p["status"]["availableReplicas"],
+                    'started': started
+                }
+                data.append(item)
+            # sorting so latest start date is first
+            data.sort(key=lambda x: x['started'], reverse=True)
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = '(list deployments): {}'.format(e)
+            self.log(err, logging.ERROR)
+            raise ServiceUnavailable(err) from e
+
+    def list_events(self, ref_kind, ref_name, *args, **kwargs):
+        try:
+            fields = {
+                "regarding.kind": ref_kind,
+                "regarding.name": ref_name
+            }
+            kwargs["fields"] = fields
+            events = self.scheduler().events.get(self.id, **kwargs).json()['items']  # noqa
+            data = []
+            for e in events:
+                item = {
+                    'reason': e['reason'],
+                    'message': e['note'],
+                    'created': e['metadata']['creationTimestamp']
+                }
+                data.append(item)
+            # sorting so latest start date is first
+            data.sort(key=lambda x: x['created'], reverse=False)
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = '(list event): {}'.format(e)
+            self.log(err, logging.ERROR)
+            raise ServiceUnavailable(err) from e
+
     def autoscale(self, proc_type, autoscale):
         """
         Set autoscale rules for the application
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index b0ee8dafc..19b44de96 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -540,6 +540,27 @@ def to_representation(self, obj):
         return obj
 
 
+class EventSerializer(serializers.BaseSerializer):
+    reason = serializers.CharField()
+    message = serializers.CharField()
+    created = serializers.DateTimeField(required=False)
+
+    def to_representation(self, obj):
+        return obj
+
+
+class PtypesSerializer(serializers.BaseSerializer):
+    name = serializers.CharField(required=False)
+    release = serializers.CharField()
+    ready = serializers.CharField()
+    up_to_date = serializers.CharField()
+    available_replicas = serializers.CharField(required=False)
+    started = serializers.DateTimeField(required=False)
+
+    def to_representation(self, obj):
+        return obj
+
+
 class AppSettingsSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.appsettings.AppSettings` model."""
 
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 1d198923b..73bf0154c 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -79,6 +79,23 @@ def restart_app(app, **kwargs):
         signals.request_finished.send(sender=task_id)
 
 
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_jitter=True,
+    retry_kwargs={'max_retries': 3}
+)
+def delete_pod(app, **kwargs):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        app.delete_pod(**kwargs)
+    except Exception as e:
+        signals.got_request_exception.send(sender=task_id)
+        raise e
+    else:
+        signals.request_finished.send(sender=task_id)
+
+
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_jitter=True,
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 898cbf045..e06cb9d1f 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -244,7 +244,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # scale worker
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'worker': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -304,7 +304,7 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale worker
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'worker': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -340,7 +340,7 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.json()['structure'], {'web': 1, 'worker': 1})
 
         # scale worker to make sure no info was lost
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'worker': 2}  # bump from 1 to 2
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -376,7 +376,7 @@ def test_build_forgotten_procfile_reject(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale worker
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'worker': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -472,7 +472,7 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # scale to zero
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index f44d45405..b87642168 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -507,7 +507,7 @@ def test_unset_limits_error(self, mock_requests):
         self.assertEqual(str(response.data["detail"]), "no-exists does not exist under limits")
         # scale up
         body = {'web': 3}
-        response = self.client.post(f"/v2/apps/{app_id}/scale", body)
+        response = self.client.post(f"/v2/apps/{app_id}/ptypes/scale", body)
         self.assertEqual(response.status_code, 204, response.data)
 
         body = {
diff --git a/rootfs/api/tests/test_event.py b/rootfs/api/tests/test_event.py
new file mode 100644
index 000000000..93eb3f864
--- /dev/null
+++ b/rootfs/api/tests/test_event.py
@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for the Drycc api app.
+
+Run the tests with "./manage.py test api"
+"""
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from django.conf import settings
+
+from api.models.app import App
+
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class EventTest(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        url = '/v2/apps'
+        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
+        self.assertEqual(response.status_code, 201, response.data)
+        self.app = App.objects.all()[0]
+
+    def tearDown(self):
+        # Restore default tags to empty string
+        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def build_deploy(self, app_id):
+        # post a new build with procfile
+        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        body = {
+            'image': 'autotest/example',
+            'sha': 'a'*40,
+            'stack': 'heroku-18',
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+    def test_events(self, mock_requests):
+        """
+        Test that config is auto-created for a new app and that
+        config can be updated using a PATCH
+        """
+        app_id = self.create_app()
+        self.build_deploy(app_id)
+
+        # list events of pod
+        url = f"/v2/apps/{app_id}/pods"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        pod_name = response.data['results'][0]["name"]
+        response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
+        url = f"/v2/apps/{app_id}/events"
+        response = self.client.get(url, {"pod_name": pod_name})
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # list events of deployment
+        url = f"/v2/apps/{app_id}/events"
+        response = self.client.get(url, {"ptype_name": f"{app_id}-web"})
+        self.assertEqual(response.status_code, 200, response.data)
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index a5d53fb1f..62ac53594 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -60,7 +60,7 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         # test setting one proc type at a time
         body = {'web': 4}
         response = self.client.post(url, body)
@@ -83,7 +83,7 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(response.data['structure']['worker'], 2)
 
         # scale down
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         # test setting two proc types at a time
         body = {'web': 2, 'worker': 1}
         response = self.client.post(url, body)
@@ -102,7 +102,7 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(response.data['structure']['worker'], 1)
 
         # scale down to 0
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 0, 'worker': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -136,7 +136,7 @@ def test_container_api(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 6}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -151,7 +151,7 @@ def test_container_api(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
 
         # scale down
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 3}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -166,7 +166,7 @@ def test_container_api(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
 
         # scale down to 0
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -204,7 +204,7 @@ def test_release(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -243,9 +243,20 @@ def test_release(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertPodContains(response.data['results'], app_id, "web", 'v4')
+
+        # describe pod
         pod_name = response.data['results'][0]["name"]
         response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
         self.assertEqual(response.status_code, 200, response.data)
+
+        # delete pods
+        response = self.client.delete(
+            f"/v2/apps/{app_id}/pods/",
+            {'pod_ids': pod_name}
+        )
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # describe no exists pod
         pod_name = "no-exists-pod-name"
         response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
         self.assertEqual(response.status_code, 404, response.data)
@@ -268,7 +279,7 @@ def test_container_errors(self, mock_requests):
         )
         # deploy
         app.pipeline(release)
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 'not_an_int'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -298,7 +309,7 @@ def test_container_str(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4, 'worker': 2}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -336,7 +347,7 @@ def test_pod_command_format(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -382,7 +393,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale to a negative number
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': -1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -390,7 +401,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(app.structure["web"], 1)
 
         # scale to something other than a number
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 'one'}
         response = self.client.post(url, body)
         app = App.objects.get(id=app_id)
@@ -398,7 +409,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(app.structure["web"], 1)
 
         # scale to something other than a number
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': [1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -406,7 +417,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(app.structure["web"], 1)
 
         # scale with a non-existent proc type
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'foo': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -414,7 +425,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual("foo" in app.structure, False)
 
         # scale up to an integer as a sanity check
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -423,7 +434,7 @@ def test_scale_errors(self, mock_requests):
 
         with mock.patch('scheduler.KubeHTTPClient.scale') as mock_kube:
             mock_kube.side_effect = KubeException('Boom!')
-            url = f"/v2/apps/{app_id}/scale"
+            url = f"/v2/apps/{app_id}/ptypes/scale"
             response = self.client.post(url, {'web': 10})
             self.assertEqual(response.status_code, 204, response.data)
             app = App.objects.get(id=app_id)
@@ -454,7 +465,7 @@ def test_admin_can_manage_other_pods(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # login as admin, scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4, 'worker': 2}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -466,7 +477,7 @@ def test_scale_without_build_should_error(self, mock_requests):
         body = {'cluster': 'autotest', 'id': app_id}
         response = self.client.post(url, body)
 
-        url = f'/v2/apps/{app_id}/scale'
+        url = f'/v2/apps/{app_id}/ptypes/scale'
         body = {'web': '1'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -667,7 +678,7 @@ def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # scale up
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 3}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -699,7 +710,7 @@ def test_scale_with_unauthorized_user_returns_403(self, mock_requests):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
 
         # scale up with unauthorized user
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
@@ -724,7 +735,7 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         }
         response = self.client.post(build_url, body)
 
-        url = f"/v2/apps/{app_id}/scale"
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -745,48 +756,6 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         pods = application.list_pods(type='web')
         self.assertEqual(len(pods), 0)
 
-    def test_restart_pods(self, mock_requests):
-        app_id = self.create_app()
-
-        # post a new build
-        build_url = f"/v2/apps/{app_id}/builds"
-        body = {
-            'image': 'autotest/example',
-            'stack': 'heroku-18',
-            'sha': 'a'*40,
-            'procfile': {
-                'web': 'node server.js',
-                'worker': 'node worker.js'
-            }
-        }
-        response = self.client.post(build_url, body)
-
-        url = f"/v2/apps/{app_id}/scale"
-        body = {'web': 4, 'worker': 8}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # setup app object
-        application = App.objects.get(id=app_id)
-
-        # restart all pods
-        response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id))
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # restart web and workers pods
-        body = {"types": "web,worker"}
-        response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id), body)
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # restart invalid ptypes
-        body = {"types": "web1"}
-        response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id), body)
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # restart only one of the web pods
-        pods = application.list_pods(type='web')
-        self.assertEqual(len(pods), 4)
-
     def test_list_pods_failure(self, mock_requests):
         """
         Listing all available pods exceptions
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
new file mode 100644
index 000000000..dd1c5785c
--- /dev/null
+++ b/rootfs/api/tests/test_ptypes.py
@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for the Drycc api app.
+
+Run the tests with "./manage.py test api"
+"""
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from django.conf import settings
+
+from api.models.app import App
+
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class PtypesTest(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        url = '/v2/apps'
+        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
+        self.assertEqual(response.status_code, 201, response.data)
+        self.app = App.objects.all()[0]
+
+    def tearDown(self):
+        # Restore default tags to empty string
+        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def build_deploy(self, app_id):
+        # post a new build with procfile
+        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        body = {
+            'image': 'autotest/example',
+            'sha': 'a'*40,
+            'stack': 'heroku-18',
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+    def test_ptype(self, mock_requests):
+        """
+        Test that config is auto-created for a new app and that
+        config can be updated using a PATCH
+        """
+        app_id = self.create_app()
+        self.build_deploy(app_id)
+
+        # list ptypes deployment
+        url = f"/v2/apps/{app_id}/ptypes"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # describe ptype deployment
+        url = f"/v2/apps/{app_id}/ptypes/{app_id}-web/describe"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+
+        ptype_name = "no-exists-ptype-name"
+        response = self.client.get(f"/v2/apps/{app_id}/ptypes/{ptype_name}/describe/")
+        self.assertEqual(response.status_code, 404, response.data)
+
+    def test_restart_ptypes(self, mock_requests):
+        app_id = self.create_app()
+
+        # post a new build
+        build_url = f"/v2/apps/{app_id}/builds"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(build_url, body)
+
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'web': 4, 'worker': 8}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # setup app object
+        application = App.objects.get(id=app_id)
+
+        # restart all ptypes deployments
+        response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id))
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # restart web and workers ptype deployments
+        body = {"types": "web,worker"}
+        response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id), body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # restart invalid ptypes
+        body = {"types": "web1"}
+        response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id), body)
+        self.assertEqual(response.status_code, 400, response.data)
+
+        # restart only one of the web ptype deployments
+        pods = application.list_pods(type='web')
+        self.assertEqual(len(pods), 4)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 9eb09bab1..69fd4b291 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -45,18 +45,34 @@
     re_path(
         r"^apps/(?P<id>{})/releases/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'get': 'list'})),
-    # restart pods
-    re_path(
-        r"^apps/(?P<id>{})/pods/restart/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'post': 'restart'})),
-    # list pods
+    # list/delete pods
     re_path(
         r"^apps/(?P<id>{})/pods/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list'})),
+        views.PodViewSet.as_view({'get': 'list', 'delete': 'delete'})),
     # describe pod
     re_path(
         r"^apps/(?P<id>{})/pods/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),
         views.PodViewSet.as_view({'get': 'describe'})),
+    # restart deployment/ptype's pods
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/restart/?$".format(settings.APP_URL_REGEX),
+        views.PtypesViewSet.as_view({'post': 'restart'})),
+    # scale ptype replcas
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/scale/?$".format(settings.APP_URL_REGEX),
+        views.PtypesViewSet.as_view({'post': 'scale'})),
+    # list ptypes
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/?$".format(settings.APP_URL_REGEX),
+        views.PtypesViewSet.as_view({'get': 'list'})),
+    # describe ptypes
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),  # noqa
+        views.PtypesViewSet.as_view({'get': 'describe'})),
+    # list events
+    re_path(
+        r"^apps/(?P<id>{})/events/?$".format(settings.APP_URL_REGEX),  # noqa
+        views.EventViewSet.as_view({'get': 'list'})),
     # application domains
     re_path(
         r"^apps/(?P<id>{})/domains/(?P<domain>\**\.?[-\._\w]+)/?$".format(settings.APP_URL_REGEX),
@@ -70,9 +86,6 @@
         views.ServiceViewSet.as_view({'post': 'create_or_update',
                                      'get': 'list', 'delete': 'delete'})),
     # application actions
-    re_path(
-        r"^apps/(?P<id>{})/scale/?$".format(settings.APP_URL_REGEX),
-        views.AppViewSet.as_view({'post': 'scale'})),
     re_path(
         r"^apps/(?P<id>{})/run/?$".format(settings.APP_URL_REGEX),
         views.AppViewSet.as_view({'post': 'run'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 38f8265da..88ef7d34b 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -26,7 +26,8 @@
 from rest_framework.parsers import MultiPartParser
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
-from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner
+from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
+    delete_pod
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
@@ -304,14 +305,6 @@ def list(self, request, *args, **kwargs):
         serializer = self.get_serializer(instance, many=True)
         return Response(serializer.data)
 
-    def scale(self, request, **kwargs):
-        app = self.get_object()
-        latest_release = app.release_set.filter(failed=False).latest()
-        if latest_release.build is not None and latest_release.state == "created":
-            raise DryccException('There is an executing pipeline, please wait')
-        scale_app.delay(self.get_object(), request.user, request.data)
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
     def run(self, request, **kwargs):
         app = self.get_object()
         command = request.data.get('command', '').split()
@@ -429,19 +422,59 @@ def post_save(self, config):
             raise DryccException(str(e)) from e
 
 
-class PodViewSet(AppResourceViewSet):
+class PodViewSet(BaseDryccViewSet):
     model = models.app.App
     serializer_class = serializers.PodSerializer
 
     def list(self, *args, **kwargs):
-        pods = self.get_app().list_pods(*args, **kwargs)
+        pods = self.get_object().list_pods(*args, **kwargs)
         data = self.get_serializer(pods, many=True).data
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
 
+    def describe(self, *args, **kwargs):
+        pod_name = kwargs["name"]
+        data = self.get_object().describe_pod(pod_name)
+        if len(data) == 0:
+            return Response(status=status.HTTP_404_NOT_FOUND)
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
+    def delete(self, request, **kwargs):
+        pod_names = request.data.get("pod_ids")
+        pod_names = pod_names.split(",")
+        for pod_name in set(pod_names):
+            delete_pod.delay(self.get_object(), **{"pod_name": pod_name})
+        return Response(status=status.HTTP_200_OK)
+
+
+class PtypesViewSet(BaseDryccViewSet):
+    model = models.app.App
+    serializer_class = serializers.PtypesSerializer
+
+    def get_queryset(self, *args, **kwargs):
+        return self.model.objects.all(*args, **kwargs)
+
+    def list(self, *args, **kwargs):
+        deploys = self.get_object().list_deployments(*args, **kwargs)
+        data = self.get_serializer(deploys, many=True).data
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
+    def describe(self, *args, **kwargs):
+        deployment_name = kwargs["name"]
+        data = self.get_object().describe_deployment(deployment_name)
+        if len(data) == 0:
+            return Response(status=status.HTTP_404_NOT_FOUND)
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
     def restart(self, request, *args, **kwargs):
-        app = self.get_app()
+        app = self.get_object()
         ptypes = []
         types = request.data.get("types", "").split(",")
         types = [ptype for ptype in set(types) if ptype != ""]
@@ -455,14 +488,32 @@ def restart(self, request, *args, **kwargs):
                 raise DryccException("process type {} is not included in procfile".
                                      format(','.join(invalid_ptypes)))
         for ptype in set(ptypes):
-            restart_app.delay(self.get_app(), **{"type": ptype})
+            restart_app.delay(app, **{"type": ptype})
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def describe(self, *args, **kwargs):
-        pod_name = kwargs["name"]
-        data = self.get_app().describe_pod(pod_name)
-        if len(data) == 0:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+    def scale(self, request, **kwargs):
+        app = self.get_object()
+        latest_release = app.release_set.filter(failed=False).latest()
+        if latest_release.build is not None and latest_release.state == "created":
+            raise DryccException('There is an executing pipeline, please wait')
+        scale_app.delay(app, request.user, request.data)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+
+class EventViewSet(AppResourceViewSet):
+    model = models.app.App
+    serializer_class = serializers.EventSerializer
+
+    def list(self, request, **kwargs):
+        ptype_name = request.query_params.get("ptype_name", None)
+        pod_name = request.query_params.get("pod_name", None)
+        if not any([ptype_name, pod_name]):
+            data = []
+        else:
+            ref_kind, ref_name = ("Pod", pod_name) if pod_name else \
+                ("Deployment", ptype_name)
+            events = self.get_app().list_events(ref_kind, ref_name)
+            data = self.get_serializer(events, many=True).data
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index 4dfae673c..12e28a685 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -726,7 +726,9 @@ def post(request, context):
     # Handle RC / RS / Deployments
     if resource_type in ['replicationcontrollers', 'replicasets', 'deployments']:
         data['status'] = {
-            'observedGeneration': 1
+            'observedGeneration': 1,
+            'readyReplicas': 1,
+            'replicas': 1
         }
         data['metadata']['generation'] = 1
 
@@ -992,7 +994,8 @@ def add_cache_item(url, resource_type, data):
 
     # Keep track of what resources exist under other resources (mostly namespace)
     # sneaky way of getting data up to the resource type without too much magic
-    cache_url = ''.join(url.partition(resource_type)[0:2])
+    index = url.rfind(resource_type)
+    cache_url = url[:index + len(resource_type)] if index != -1 else url
     items = cache.get(cache_url, [])
     if url not in items:
         items.append(url)
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index 21745ba40..0d59f5d85 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -439,12 +439,12 @@ def _check_for_failed_events(self, namespace, labels):
         response = self.rs.get(namespace, labels=labels)
         data = response.json()
         fields = {
-            'involvedObject.kind': 'ReplicaSet',
-            'involvedObject.name': data['items'][0]['metadata']['name'],
-            'involvedObject.namespace': namespace,
-            'involvedObject.uid': data['items'][0]['metadata']['uid'],
+            'regarding.kind': 'ReplicaSet',
+            'regarding.name': data['items'][0]['metadata']['name'],
+            'regarding.namespace': namespace,
+            'regarding.uid': data['items'][0]['metadata']['uid'],
         }
-        events_list = self.ns.events(namespace, fields=fields).json()
+        events_list = self.ev.get(namespace, fields=fields).json()
         events = events_list.get('items', [])
         if events is not None and len(events) != 0:
             for event in events:
@@ -458,10 +458,11 @@ def _get_formatted_messages(events):
         """
         Format each event by string and join all events to one string
         """
-        message_format = 'Message:{message}, lastTimestamp:{lastTimestamp}, reason: {reason}, count: {count}'  # noqa
+        message_format = 'Message: {note}, creationTimestamp: {creation_timestamp}, reason: {reason}'  # noqa
         output = []
         for event in events:
-            output.append(message_format.format(**event))
+            creation_timestamp = event["metadata"]["creationTimestamp"]
+            output.append(message_format.format(creation_timestamp=creation_timestamp, **event))
         return '\n'.join(output)
 
     def _get_deploy_steps(self, batches, tags):
diff --git a/rootfs/scheduler/resources/events.py b/rootfs/scheduler/resources/events.py
index 3187defab..749098ac8 100644
--- a/rootfs/scheduler/resources/events.py
+++ b/rootfs/scheduler/resources/events.py
@@ -9,8 +9,9 @@
 class Events(Resource):
     """
     Events resource.
-    Warning! Used ONLY for testing purposes
     """
+    api_prefix = 'apis'
+    api_version = 'events.k8s.io/v1'
     short_name = 'ev'
 
     def create(self, namespace, name, message, **kwargs):
@@ -18,7 +19,6 @@ def create(self, namespace, name, message, **kwargs):
         data = {
             'kind': 'Event',
             'apiVersion': self.api_version,
-            'count': kwargs.get('count', 1),
             'metadata': {
                 'creationTimestamp': datetime.now().strftime(DATETIME_FORMAT),
                 'namespace': namespace,
@@ -26,15 +26,10 @@ def create(self, namespace, name, message, **kwargs):
                 'resourceVersion': kwargs.get('resourceVersion', ''),
                 'uid': str(uuid.uuid4()),
             },
-            'message': message,
+            'note': message,
             'type': kwargs.get('type', 'Normal'),
-            'firstTimestamp': datetime.now().strftime(DATETIME_FORMAT),
-            'lastTimestamp': datetime.now().strftime(DATETIME_FORMAT),
             'reason': kwargs.get('reason', ''),
-            'source': {
-                'component': kwargs.get('component', ''),
-            },
-            'involvedObject': kwargs.get('involvedObject', {})
+            'regarding': kwargs.get('regarding', {})
         }
 
         response = self.http_post(url, json=data)
@@ -42,3 +37,18 @@ def create(self, namespace, name, message, **kwargs):
             raise KubeHTTPException(response, 'create Event for namespace {}'.format(namespace))  # noqa
 
         return response
+
+    def get(self, namespace, ignore_exception=False, **kwargs):
+        """
+        Fetch Events
+        """
+        url = '/namespaces/{}/events'
+        args = [namespace]
+        message = 'get Events in Namespace "{}"'
+
+        url = self.api(url, *args)
+        response = self.http_get(url, params=self.query_params(**kwargs))
+        if self.unhealthy(response.status_code) and not ignore_exception:
+            args.reverse()  # error msg is in reverse order
+            raise KubeHTTPException(response, message, *args)
+        return response
diff --git a/rootfs/scheduler/resources/namespace.py b/rootfs/scheduler/resources/namespace.py
index ed420b9c5..120c9ee34 100644
--- a/rootfs/scheduler/resources/namespace.py
+++ b/rootfs/scheduler/resources/namespace.py
@@ -52,11 +52,3 @@ def delete(self, namespace):
             raise KubeHTTPException(response, 'delete Namespace "{}"', namespace)
 
         return response
-
-    def events(self, namespace, **kwargs):
-        url = self.api("/namespaces/{}/events", namespace)
-        response = self.http_get(url, params=self.query_params(**kwargs))
-        if self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, "get Events in Namespace {}", namespace)
-
-        return response
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 3b68a88db..510a1890f 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -484,7 +484,7 @@ def pending_status(self, pod):
                     return reason, message
 
                 event = events.pop()
-                return event['reason'], event['message']
+                return event['reason'], event['note']
 
             return reason, message
 
@@ -495,17 +495,16 @@ def events(self, pod):
         """Process events for a given Pod to find if Pulling is happening, among other events"""
         # fetch all events for this pod
         fields = {
-            'involvedObject.name': pod['metadata']['name'],
-            'involvedObject.namespace': pod['metadata']['namespace'],
-            'involvedObject.uid': pod['metadata']['uid']
+            'regarding.name': pod['metadata']['name'],
+            'regarding.namespace': pod['metadata']['namespace'],
+            'regarding.uid': pod['metadata']['uid']
         }
-        events = self.ns.events(
+        events = self.ev.get(
             pod['metadata']['namespace'], fields=fields).json()['items']
         if not events:
             events = []
         # make sure that events are sorted
-        events.sort(
-            key=lambda x: x['lastTimestamp'] if x['lastTimestamp'] else "")
+        events.sort(key=lambda x: x['metadata']['creationTimestamp'] if x['metadata']['creationTimestamp'] else "")  # noqa
         return events
 
     def _handle_pod_errors(self, pod, reason, message):
@@ -547,7 +546,7 @@ def _handle_pod_errors(self, pod, reason, message):
                     # only show a given error once
                     event_errors.pop(event['reason'])
                     # strip out whitespaces on either side
-                    message = "\n".join([x.strip() for x in event['message'].split("\n")])
+                    message = "\n".join([x.strip() for x in event['note'].split("\n")])
                     messages.append(message)
 
         if messages:
diff --git a/rootfs/scheduler/tests/test_deployments.py b/rootfs/scheduler/tests/test_deployments.py
index 40cb6eb53..c465f9835 100644
--- a/rootfs/scheduler/tests/test_deployments.py
+++ b/rootfs/scheduler/tests/test_deployments.py
@@ -342,20 +342,20 @@ def test_check_for_failed_events(self):
         deployment = self.scheduler.deployment.get(self.namespace, deploy_name).json()
         response = self.scheduler.rs.get(self.namespace, labels=deployment['metadata']['labels'])
         rs = response.json()
-        involved_object = {
-            'involvedObject.kind': 'ReplicaSet',
-            'involvedObject.name': rs['items'][0]['metadata']['name'],
-            'involvedObject.namespace': self.namespace,
-            'involvedObject.uid': rs['items'][0]['metadata']['uid'],
+        regarding = {
+            'regarding.kind': 'ReplicaSet',
+            'regarding.name': rs['items'][0]['metadata']['name'],
+            'regarding.namespace': self.namespace,
+            'regarding.uid': rs['items'][0]['metadata']['uid'],
         }
         message = 'Quota exeeded'
         self.scheduler.ev.create(self.namespace,
                                  '{}'.format(generate_random_name()),
                                  message,
                                  type='Warning',
-                                 involved_object=involved_object,
+                                 regarding=regarding,
                                  reason='FailedCreate')
         with self.assertRaisesRegex(KubeException,
-                                    'Message:{}.*'.format(message)):
+                                    'Message: {}.*'.format(message)):
             self.scheduler.deployment._check_for_failed_events(self.namespace,
                                                                labels=deployment['metadata']['labels'])  # noqa
diff --git a/rootfs/scheduler/tests/test_events.py b/rootfs/scheduler/tests/test_events.py
new file mode 100644
index 000000000..89b17e276
--- /dev/null
+++ b/rootfs/scheduler/tests/test_events.py
@@ -0,0 +1,38 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with './manage.py test scheduler'
+"""
+from scheduler.tests import TestCase
+
+
+class EventTest(TestCase):
+    """Tests scheduler pod calls"""
+
+    def create_event(self, namespace, name, **kwargs):
+        """
+        Helper function to create and verify a events on the namespace
+        """
+        self.scheduler.ns.create(namespace)
+
+        message = "Scaled down replica set test-869947c55f to 1 from 2"
+        # these are all required even if it is kwargs...
+        kwargs = {
+            'reason': 'ScalingReplicaSet',
+            'type': 'Normal'
+        }
+        return self.scheduler.events.create(
+            namespace,
+            name,
+            message,
+            **kwargs
+        )
+
+    def test_create(self):
+        response = self.create_event("test-event", "test-event")
+        self.assertEqual(response.status_code, 201)
+
+    def test_get_events(self):
+        # test success
+        response = self.scheduler.events.get("test-event", "test-event")
+        self.assertEqual(response.status_code, 200)

From d04fb844c7ac52cdbea709c150011d30492c4306 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 9 Jul 2024 18:10:29 +0800
Subject: [PATCH 035/200] fix(controller): list_deployments status info error

---
 rootfs/api/models/app.py        | 8 ++++----
 rootfs/api/tests/test_pods.py   | 2 +-
 rootfs/api/tests/test_ptypes.py | 2 +-
 rootfs/api/views.py             | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 627b83d55..efc8e1815 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -484,11 +484,11 @@ def list_deployments(self, *args, **kwargs):
                     'name': labels['type'],
                     'release': labels['version'],
                     'ready': "%s/%s" % (
-                        p["status"]["readyReplicas"],
-                        p["status"]["replicas"],
+                        p["status"].get("readyReplicas", 0),
+                        p["status"].get("replicas", 0),
                     ),
-                    'up_to_date': p["status"]["updatedReplicas"],
-                    'available_replicas': p["status"]["availableReplicas"],
+                    'up_to_date': p["status"].get("updatedReplicas", 0),
+                    'available_replicas': p["status"].get("availableReplicas", 0),
                     'started': started
                 }
                 data.append(item)
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index 62ac53594..6abf99687 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -259,7 +259,7 @@ def test_release(self, mock_requests):
         # describe no exists pod
         pod_name = "no-exists-pod-name"
         response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
-        self.assertEqual(response.status_code, 404, response.data)
+        self.assertEqual(response.status_code, 400, response.data)
 
     def test_container_errors(self, mock_requests):
         app_id = self.create_app()
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
index dd1c5785c..15b2ab75a 100644
--- a/rootfs/api/tests/test_ptypes.py
+++ b/rootfs/api/tests/test_ptypes.py
@@ -73,7 +73,7 @@ def test_ptype(self, mock_requests):
 
         ptype_name = "no-exists-ptype-name"
         response = self.client.get(f"/v2/apps/{app_id}/ptypes/{ptype_name}/describe/")
-        self.assertEqual(response.status_code, 404, response.data)
+        self.assertEqual(response.status_code, 400, response.data)
 
     def test_restart_ptypes(self, mock_requests):
         app_id = self.create_app()
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 88ef7d34b..83fc58377 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -437,7 +437,7 @@ def describe(self, *args, **kwargs):
         pod_name = kwargs["name"]
         data = self.get_object().describe_pod(pod_name)
         if len(data) == 0:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+            raise DryccException("this process not found")
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
@@ -468,7 +468,7 @@ def describe(self, *args, **kwargs):
         deployment_name = kwargs["name"]
         data = self.get_object().describe_deployment(deployment_name)
         if len(data) == 0:
-            return Response(status=status.HTTP_404_NOT_FOUND)
+            raise DryccException("this ptype not found")
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)

From 4ded2daa0509557d38f1d9446066d700dda802f4 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 9 Jul 2024 18:14:50 +0800
Subject: [PATCH 036/200] chore(charts): change filer settings

---
 charts/controller/templates/_helpers.tpl | 4 ++++
 charts/controller/values.yaml            | 3 +++
 rootfs/api/filer.py                      | 2 +-
 rootfs/api/settings/production.py        | 1 +
 4 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 5818dae74..31b65bc92 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -16,6 +16,10 @@ env:
   value: "{{ .Values.global.registrySecretPrefix }}"
 - name: "IMAGE_PULL_POLICY"
   value: "{{ .Values.appImagePullPolicy }}"
+- name: "DRYCC_FILER_IMAGE"
+  value: "{{ (tpl .Values.filerImage .) }}"
+- name: "DRYCC_FILER_IMAGE_PULL_POLICY"
+  value: "{{ (tpl .Values.filerImagePullPolicy .) }}"
 - name: "KUBERNETES_CLUSTER_DOMAIN"
   value: "{{ .Values.global.clusterDomain }}"
 {{- if (.Values.appStorageClass) }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 592465ebc..170dbda7e 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -31,6 +31,9 @@ registrationMode: "admin_only"
 k8sApiVerifyTls: "true"
 # Set app image pull policy
 appImagePullPolicy: "Always"
+# Filer settings image and image pull policy
+filerImage: "registry.drycc.cc/drycc/filer:canary"
+filerImagePullPolicy: "Always"
 # Set storageClassName. It is used for application.
 appStorageClass: "{{ .Values.storage.csi.storageClassName }}"
 # Set runtimeClassName. It is used for application.
diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
index 211b69a73..84a1eab8e 100644
--- a/rootfs/api/filer.py
+++ b/rootfs/api/filer.py
@@ -53,7 +53,7 @@ def get_server(self):
             "labels": {"app": self.app_id, "pod": pod_name, "volume": self.volume.name},
             "app_type": "filer", "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
             "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
-            "restart_policy": "Never", "image_pull_policy": "Always",
+            "restart_policy": "Never", "image_pull_policy": settings.DRYCC_FILER_IMAGE_PULL_POLICY,
         })
         address = self.scheduler.pod.get(self.app_id, pod_name).json()["status"]["podIP"]
         return {"address": address, "username": username, "password": password}
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 823fcba19..5cdb20ef9 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -303,6 +303,7 @@
 # Drycc filer image
 # Provide get and put operations for `drycc volumes:client`
 DRYCC_FILER_IMAGE = os.environ.get('DRYCC_FILER_IMAGE', 'registry.drycc.cc/drycc/filer:canary')
+DRYCC_FILER_IMAGE_PULL_POLICY = os.environ.get('DRYCC_FILER_IMAGE_PULL_POLICY', 'IfNotPresent')
 DRYCC_FILER_DURATION = int(os.environ.get('DRYCC_FILER_DURATION', '3600'))
 DRYCC_FILER_WAITTIME = int(os.environ.get('DRYCC_FILER_WAITTIME', '1200'))
 

From cd99807d7b619d38fe22bf6ec16c5e10372163ec Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 10 Jul 2024 10:10:23 +0800
Subject: [PATCH 037/200] chore(controller): change upgrade job helm hook

---
 .../templates/controller-job-upgrade.yaml     | 11 +---
 rootfs/api/models/gateway.py                  | 51 +++++++++----------
 2 files changed, 26 insertions(+), 36 deletions(-)

diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index 11663eee9..0240f2cb7 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -1,18 +1,13 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: drycc-controller-job-upgrade-{{ now | date "20060102150405" }}
-  labels:
-    heritage: drycc
+  name: drycc-controller-job-upgrade
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
     helm.sh/hook: post-install,pre-upgrade,pre-rollback
-    helm.sh/hook-delete-policy: hook-succeeded
+    helm.sh/hook-delete-policy: before-hook-creation
 spec:
   template:
-    metadata:
-      labels: {{- include "common.labels.standard" . | nindent 8 }}
-        app: drycc-controller-job-upgrade
     spec:
       initContainers:
       - name: drycc-controller-job-upgrade-init
@@ -53,5 +48,3 @@ spec:
         configMap:
           name: controller-config
       restartPolicy: Never
-  backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 911a6b6f8..704f515fa 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -324,49 +324,46 @@ def _check_parent(self, gateway_name, port):
         return True, ""
 
     def _refresh_to_k8s(self, rules, parent_refs):
+        manifest = {
+            "rules": rules,
+            "hostnames": self.hostnames,
+            "parent_refs": parent_refs,
+        }
         try:
             k8s_route = getattr(self.scheduler(), self.kind.lower())
             try:
                 data = k8s_route.get(self.app.id, self.name).json()
-                k8s_route.patch(self.app.id, self.name, **{
-                    "rules": rules,
-                    "hostnames": self.hostnames,
-                    "parent_refs": parent_refs,
-                    "version": data["metadata"]["resourceVersion"],
-                })
+                manifest.update({"version": data["metadata"]["resourceVersion"]})
+                k8s_route.patch(self.app.id, self.name, **manifest)
             except KubeException:
-                k8s_route.create(self.app.id, self.name, **{
-                    "rules": rules,
-                    "hostnames": self.hostnames,
-                    "parent_refs": parent_refs,
-                })
+                k8s_route.create(self.app.id, self.name, **manifest)
         except KubeException as e:
             raise ServiceUnavailable(
                 f'Kubernetes {self.kind.lower()} could not be created') from e
 
     def _https_enforced_to_k8s(self, parent_refs):
-        rules = {
-            "filters": [{
-                "type": "RequestRedirect",
-                "requestRedirect": {
-                    "port": DEFAULT_HTTPS_PORT, "scheme": "https", "statusCode": 301
-                }
-            }]
+        manifest = {
+            "rules": [{
+                "filters": [{
+                    "type": "RequestRedirect",
+                    "requestRedirect": {
+                        "port": DEFAULT_HTTPS_PORT, "scheme": "https", "statusCode": 301
+                    }
+                }]
+            }],
+            "hostnames": self.hostnames,
+            "parent_refs": parent_refs,
         }
         try:
             try:
                 data = self.scheduler().httproute.get(
                     self.app.id, self._https_redirect_name).json()
-                self.scheduler().httproute.patch(self.app.id, self._https_redirect_name, **{
-                    "rules": rules,
-                    "parent_refs": parent_refs,
-                    "version": data["metadata"]["resourceVersion"],
-                })
+                manifest.update({"version": data["metadata"]["resourceVersion"]})
+                self.scheduler().httproute.patch(
+                    self.app.id, self._https_redirect_name, **manifest)
             except KubeException:
-                self.scheduler().httproute.create(self.app.id, self._https_redirect_name, **{
-                    "rules": rules,
-                    "parent_refs": parent_refs,
-                })
+                self.scheduler().httproute.create(
+                    self.app.id, self._https_redirect_name, **manifest)
         except KubeException as e:
             raise ServiceUnavailable(
                 f'Kubernetes {self.kind.lower()} could not be created') from e

From 25b9a5eb1bf8fc920aac13cb2c5f469fb7ef6521 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 10 Jul 2024 17:20:11 +0800
Subject: [PATCH 038/200] fix(filer): get big data error

---
 rootfs/api/views.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 83fc58377..d0818aa5c 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -895,10 +895,12 @@ def list(self, request, **kwargs):
     def retrieve(self, request, **kwargs):
         path = self.kwargs.get('path', '')
         client = self.get_client()
+        chunk_size = 1024 * 1024 * 2
         response = client.get(path, stream=True, params={"action": "get"})
         return FileResponse(
             status=response.status_code,
-            streaming_content=utils.iter_to_aiter(response.iter_content()),
+            headers=response.headers,
+            streaming_content=utils.iter_to_aiter(response.iter_content(chunk_size=chunk_size)),
         )
 
     def create(self, request, **kwargs):

From 123bb97f400f88a2d2dc4d99d12bf4a7f2c4b427 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 11 Jul 2024 00:49:01 +0800
Subject: [PATCH 039/200] feat(config): set config to celery

---
 rootfs/api/tasks.py               | 32 +++++++++++++++++++++++++++++
 rootfs/api/tests/test_config.py   | 10 +++++++--
 rootfs/api/views.py               | 26 +++--------------------
 rootfs/scheduler/resources/pod.py | 34 +++++++++++++++++++------------
 4 files changed, 64 insertions(+), 38 deletions(-)

diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 73bf0154c..2056462cb 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -62,6 +62,38 @@ def run_pipeline(release, *args, **kwargs):
         signals.request_finished.send(sender=task_id)
 
 
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_kwargs={'max_retries': None}
+)
+def run_deploy(release, config):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        if release.build is not None:
+            procfile_types = set()
+            for field, diff in config.diff().items():
+                if field in config.procfile_fields:
+                    for value in diff.values():
+                        procfile_types.update(value.keys())
+            # all_diff_fields changed, deploy all.
+            procfile_types = procfile_types if procfile_types else None
+            config.app.deploy(release, procfile_types)
+        release.state = "succeed"
+        release.save()
+    except Exception as e:
+        release.state = "crashed"
+        release.failed = True
+        if release.summary:
+            release.summary += " "
+        release.summary += "{} deployed a config that failed".format(release.owner)
+        # Get the exception that has occured
+        release.exception = "error: {}".format(str(e))
+        release.save()
+    finally:
+        signals.request_finished.send(sender=task_id)
+
+
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_jitter=True,
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index b87642168..826b8af11 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -312,7 +312,11 @@ def test_config_deploy_failure(self, mock_requests):
             url = f'/v2/apps/{app_id}/config'
             body = {'values': json.dumps({'test': "testvalue"})}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400)
+            self.assertEqual(response.status_code, 201)
+            app = App.objects.get(id=app_id)
+            release = app.release_set.latest()
+            self.assertEqual(release.failed, True)
+            self.assertEqual(release.exception, "error: Boom!")
 
     def test_invalid_config_keys(self, mock_requests):
         """Test that invalid config keys are rejected.
@@ -416,8 +420,10 @@ def test_config_failures(self, mock_requests):
             url = f'/v2/apps/{app_id}/config'
             body = {'values': json.dumps({'test': "testvalue"})}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400)
+            self.assertEqual(response.status_code, 201)
             self.assertEqual(app.release_set.latest().version, 4)
+            self.assertEqual(app.release_set.latest().failed, True)
+            self.assertEqual(app.release_set.latest().exception, "error: Boom!")
             self.assertEqual(app.release_set.filter(failed=False).latest().version, 3)
 
         # create a build to see that the new release is created with the last successful config
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index d0818aa5c..113790a8b 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -27,7 +27,7 @@
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
-    delete_pod
+    delete_pod, run_deploy
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
@@ -394,29 +394,9 @@ def post_save(self, config):
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
-            if release.build is not None:
-                procfile_types = set()
-                for field, diff in config.diff().items():
-                    if field in config.procfile_fields:
-                        for value in diff.values():
-                            procfile_types.update(value.keys())
-                # all_diff_fields changed, deploy all.
-                procfile_types = procfile_types if procfile_types else None
-                config.app.deploy(release, procfile_types)
-            release.state = "succeed"
-            release.save()
+            run_deploy.delay(release, config)
         except Exception as e:
-            if 'release' in locals():
-                release.state = "crashed"
-                release.failed = True
-                if release.summary:
-                    release.summary += " "
-                release.summary += "{} deployed a config that failed".format(self.request.user)
-                # Get the exception that has occured
-                release.exception = "error: {}".format(str(e))
-                release.save()
-            else:
-                config.delete()
+            config.delete()
             if isinstance(e, AlreadyExists):
                 raise
             raise DryccException(str(e)) from e
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 510a1890f..18b5ab9f5 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -749,33 +749,41 @@ def _handle_not_ready_pods(self, namespace, labels):
                     message = "\n".join([x.strip() for x in event['message'].split("\n")])
                     raise KubeException(message)
 
+    def _get_probe_timeout(self, probe):
+        period_seconds = probe.get('periodSeconds', 10)
+        timeout_seconds = probe.get('timeoutSeconds', 1)
+        failure_threshold = probe.get('failureThreshold', 3)
+        success_threshold = probe.get('successThreshold', 1)
+        initial_delay_seconds = probe.get('initialDelaySeconds', 0)
+        if period_seconds > initial_delay_seconds:
+            timeout = 0
+        else:
+            timeout = initial_delay_seconds
+        timeout += (period_seconds + timeout_seconds) * failure_threshold
+        timeout += (period_seconds + timeout_seconds) * success_threshold
+        return int(timeout)
+
     def deploy_probe_timeout(self, timeout, namespace, labels, containers):
         """
         Added in additional timeouts based on readiness and liveness probe
 
         Uses the max of the two instead of combining them as the checks are stacked.
         """
-
         container_name = '{}-{}'.format(labels['app'], labels['type'])
         container = self.pod.find_container(container_name, containers)
 
         # get health info from container
         added_timeout = []
         if 'readinessProbe' in container:
-            # If there is initial delay on the readiness check then timeout needs to be higher
-            # this is to account for kubernetes having readiness check report as failure until
-            # the initial delay period is up
-            added_timeout.append(int(container['readinessProbe'].get('initialDelaySeconds', 50)))
-
+            added_timeout.append(self._get_probe_timeout(container['readinessProbe']))
         if 'livenessProbe' in container:
-            # If there is initial delay on the readiness check then timeout needs to be higher
-            # this is to account for kubernetes having liveness check report as failure until
-            # the initial delay period is up
-            added_timeout.append(int(container['livenessProbe'].get('initialDelaySeconds', 50)))
-
+            added_timeout.append(self._get_probe_timeout(container['livenessProbe']))
         if added_timeout:
             delay = max(added_timeout)
-            self.log(namespace, "adding {}s on to the original {}s timeout to account for the initial delay specified in the liveness / readiness probe".format(delay, timeout))  # noqa
+            self.log(
+                namespace,
+                "adding {}s on to the original {}s timeout for the probe".format(delay, timeout))
             timeout += delay
-
+        if 'startupProbe' in container:
+            timeout += self._get_probe_timeout(container['startupProbe'])
         return timeout

From 45cb63c9604e24e2c373542b2c689b3897321f53 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 11 Jul 2024 16:40:52 +0800
Subject: [PATCH 040/200] chore(deps): bump django from 4.2.11 to 4.2.14

---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 22ad6e9e4..44cb0260e 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.11
+django==4.2.14
 channels==4.1.0
 aiohttp==3.9.4
 django-cors-headers==4.3.1

From 3d21898f60b7a09f29277d296b01dbe38692069e Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 11 Jul 2024 23:16:58 +0800
Subject: [PATCH 041/200] feat(tags): add ptype tags

---
 rootfs/api/models/app.py                  |  6 +-
 rootfs/api/models/config.py               | 71 +++++++++++------------
 rootfs/api/serializers/__init__.py        | 66 ++++++++++-----------
 rootfs/api/settings/production.py         |  2 -
 rootfs/api/settings/testing.py            |  2 -
 rootfs/api/tests/test_config.py           | 37 ------------
 rootfs/api/tests/test_event.py            |  3 -
 rootfs/api/tests/test_ptypes.py           |  3 -
 rootfs/api/tests/test_resource.py         |  3 -
 rootfs/api/tests/test_tags.py             | 44 +++++++-------
 rootfs/api/tests/test_volume.py           |  3 -
 rootfs/api/tests/test_workflow_manager.py |  2 -
 rootfs/scheduler/resources/deployment.py  |  2 -
 rootfs/scheduler/resources/pod.py         |  6 +-
 14 files changed, 92 insertions(+), 158 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index efc8e1815..0f0cec4ee 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -714,7 +714,7 @@ def _deploy(self, deploys, procfile_types, prev_release,
         # Sort deploys so routable comes first
         deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
         # Check if any proc type has a Deployment in progress
-        self._check_deployment_in_progress(deploys, release, force_deploy)
+        self._check_deployment_in_progress(deploys, force_deploy)
 
         try:
             tasks = []
@@ -969,7 +969,7 @@ def _verify_tcp_health(self, service, **kwargs):
                 else:
                     time.sleep(3)
 
-    def _check_deployment_in_progress(self, deploys, release, force_deploy=False):
+    def _check_deployment_in_progress(self, deploys, force_deploy=False):
         if force_deploy:
             return
         for scale_type, kwargs in deploys.items():
@@ -1127,7 +1127,7 @@ def _gather_app_settings(self, release, app_settings, procfile_type, replicas, v
         healthcheck = config.healthcheck.get(procfile_type, {})
         volumes, volume_mounts = self._get_volumes_and_mounts(procfile_type, volumes)
         return {
-            'tags': config.tags,
+            'tags': config.tags.get(procfile_type, {}),
             'envs': envs,
             'registry': config.registry,
             'replicas': replicas,
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 8eba125cc..892d11454 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -1,6 +1,4 @@
-import json
 import logging
-from django.conf import settings
 from django.db import models
 from django.contrib.auth import get_user_model
 from api.utils import dict_diff
@@ -43,20 +41,20 @@ def __str__(self):
 
     def previous(self):
         """
-        Return the previous Release to this one.
+        Return the previous Config to this one.
 
-        :return: the previous :class:`Release`, or None
+        :return: the previous :class:`Config`, or None
         """
         configs = self.app.config_set
         if self.pk:
             configs = configs.exclude(pk=self.pk)
 
         try:
-            # Get the Release previous to this one
-            prev_release = configs.latest()
-        except Release.DoesNotExist:
-            prev_release = None
-        return prev_release
+            # Get the Config previous to this one
+            prev_config = configs.latest()
+        except Config.DoesNotExist:
+            prev_config = None
+        return prev_config
 
     def diff(self, config=None):
         old_config = config if config else self.previous()
@@ -76,7 +74,7 @@ def save(self, **kwargs):
                 # usually means a totally new app
                 previous_config = self.app.config_set.latest()
 
-            for attr in ['tags', 'registry', 'values', 'lifecycle_post_start',
+            for attr in ['registry', 'values', 'lifecycle_post_start',
                          'lifecycle_pre_stop', 'termination_grace_period']:
                 data = getattr(previous_config, attr, {}).copy()
                 new_data = getattr(self, attr, {}).copy()
@@ -88,7 +86,7 @@ def save(self, **kwargs):
             self._set_registry()
             self._set_tags(previous_config)
         except Config.DoesNotExist:
-            self._set_tags({'tags': {}})
+            self._set_tags(previous_config={'tags': {}})
 
         return super(Config, self).save(**kwargs)
 
@@ -117,34 +115,31 @@ def _set_registry(self):
 
     def _set_tags(self, previous_config):
         """verify the tags exist on any nodes as labels"""
-        if not self.tags:
-            if settings.DRYCC_DEFAULT_CONFIG_TAGS:
-                try:
-                    tags = json.loads(settings.DRYCC_DEFAULT_CONFIG_TAGS)
-                    self.tags = tags
-                except json.JSONDecodeError as e:
-                    logger.exception(e)
-                    return
+        data = getattr(previous_config, 'tags', {}).copy()
+        new_data = getattr(self, 'tags', {}).copy()
+        # remove config keys if a null value is provided
+        for procfile_type, values in new_data.items():
+            if not values:
+                # error if unsetting non-existing key
+                if procfile_type not in data:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(procfile_type, 'tags'))
+                data.pop(procfile_type)
             else:
-                return
-
-        # Get all nodes with label selectors
-        nodes = self.scheduler().node.get(labels=self.tags).json()
-        if nodes['items']:
-            return
-
-        labels = ['{}={}'.format(key, value) for key, value in self.tags.items()]
-        message = 'No nodes matched the provided labels: {}'.format(', '.join(labels))
-
-        # Find out if there are any other tags around
-        old_tags = getattr(previous_config, 'tags')
-        if old_tags:
-            old = ['{}={}'.format(key, value) for key, value in old_tags.items()]
-            new = set(labels) - set(old)
-            if new:
-                message += ' - Addition of {} is the cause'.format(', '.join(new))
-
-        raise DryccException(message)
+                if not self.scheduler().node.get(labels=values).json()['items']:
+                    labels = ['{}={}'.format(key, value) for key, value in values.items()]
+                    message = 'No nodes matched the provided labels: {}'.format(', '.join(labels))
+                    # Find out if there are any other tags around
+                    old_tags = previous_config.tags.get(procfile_type, {})
+                    if old_tags:
+                        old = ['{}={}'.format(key, value) for key, value in old_tags.items()]
+                        new = set(labels) - set(old)
+                        if new:
+                            message += ' - Addition of {} is the cause'.format(', '.join(new))
+                    raise DryccException(message)
+                data[procfile_type] = self._merge_data(
+                    'tags', data.get(procfile_type, {}), values)
+        setattr(self, 'tags', data)
 
     def _set_limits(self, previous_config):
         data = getattr(previous_config, 'limits', {}).copy()
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 19b44de96..eb13a8f7f 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -286,8 +286,7 @@ def validate_values(data):
     @classmethod
     def validate_typed_values(cls, data):
         for procfile_type, values in data.items():
-            if not re.match(PROCTYPE_MATCH, procfile_type):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+            validate_procfile_type(procfile_type)
             if values is None:  # use NoneType to unset an item
                 continue
             cls.validate_values(values)
@@ -297,8 +296,7 @@ def validate_typed_values(cls, data):
     def validate_limits(data):
         req_plan_ids = []
         for procfile_type, plan_id in data.items():
-            if not re.match(PROCTYPE_MATCH, procfile_type):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+            validate_procfile_type(procfile_type)
             if plan_id is not None:
                 req_plan_ids.append(plan_id)
         plan_ids = [plan.id for plan in models.limit.LimitPlan.objects.filter(
@@ -311,8 +309,7 @@ def validate_limits(data):
     @staticmethod
     def validate_termination_grace_period(data):
         for procfile_type, value in data.items():
-            if not re.match(PROCTYPE_MATCH, procfile_type):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+            validate_procfile_type(procfile_type)
             if value is None:  # use NoneType to unset an item
                 continue
             timeout = re.match(TERMINATION_GRACE_PERIOD_MATCH, str(value))
@@ -324,37 +321,38 @@ def validate_termination_grace_period(data):
 
     @staticmethod
     def validate_tags(data):
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
-                continue
-
-            # split key into a prefix and name
-            if '/' in key:
-                prefix, name = key.split('/')
-            else:
-                prefix, name = None, key
+        for procfile_type, values in data.items():
+            validate_procfile_type(procfile_type)
+            for key, value in values.items():
+                if value is None:  # use NoneType to unset an item
+                    continue
 
-            # validate optional prefix
-            if prefix:
-                if len(prefix) > 253:
-                    raise serializers.ValidationError(
-                        "Tag key prefixes must 253 characters or less.")
+                # split key into a prefix and name
+                if '/' in key:
+                    prefix, name = key.split('/')
+                else:
+                    prefix, name = None, key
 
-                for part in prefix.split('/'):
-                    if not re.match(TAGVAL_MATCH, part):
+                # validate optional prefix
+                if prefix:
+                    if len(prefix) > 253:
                         raise serializers.ValidationError(
-                            "Tag key prefixes must be DNS subdomains.")
+                            "Tag key prefixes must 253 characters or less.")
 
-            # validate required name
-            if not re.match(TAGVAL_MATCH, name):
-                raise serializers.ValidationError(
-                    "Tag keys must be alphanumeric or \"-_.\", and 1-63 characters.")
+                    for part in prefix.split('/'):
+                        if not re.match(TAGVAL_MATCH, part):
+                            raise serializers.ValidationError(
+                                "Tag key prefixes must be DNS subdomains.")
 
-            # validate value if it isn't empty
-            if value and not re.match(TAGVAL_MATCH, str(value)):
-                raise serializers.ValidationError(
-                    "Tag values must be alphanumeric or \"-_.\", and 1-63 characters.")
+                # validate required name
+                if not re.match(TAGVAL_MATCH, name):
+                    raise serializers.ValidationError(
+                        "Tag keys must be alphanumeric or \"-_.\", and 1-63 characters.")
 
+                # validate value if it isn't empty
+                if value and not re.match(TAGVAL_MATCH, str(value)):
+                    raise serializers.ValidationError(
+                        "Tag values must be alphanumeric or \"-_.\", and 1-63 characters.")
         return data
 
     @staticmethod
@@ -371,8 +369,7 @@ def validate_registry(data):
     @staticmethod
     def validate_healthcheck(data):
         for procfile_type, healthcheck in data.items():
-            if not re.match(PROCTYPE_MATCH, procfile_type):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+            validate_procfile_type(procfile_type)
             if healthcheck is None:
                 continue
             for key, value in healthcheck.items():
@@ -630,8 +627,7 @@ def validate_path(data):
         logger.debug(f"mount validate_path data: {data}")
         new_data = {}
         for key, value in data.items():
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+            validate_procfile_type(key)
             if value is None:  # use NoneType to unset an item
                 new_data[key] = value
                 continue
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 5cdb20ef9..ce173f110 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -361,8 +361,6 @@
 
 DRYCC_APP_POD_EXEC_TIMEOUT = int(os.environ.get('DRYCC_APP_POD_EXEC_TIMEOUT', "3600"))
 
-DRYCC_DEFAULT_CONFIG_TAGS = os.environ.get('DRYCC_DEFAULT_CONFIG_TAGS', '')
-
 KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT = os.environ.get(
     'KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', None)
 
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index 41b5bdb57..48cfab261 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -43,8 +43,6 @@
 KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(
     os.environ.get('KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 2))
 
-DRYCC_DEFAULT_CONFIG_TAGS = os.environ.get('DRYCC_DEFAULT_CONFIG_TAGS', '')
-
 DRYCC_APP_STORAGE_CLASS = os.environ.get('DRYCC_APP_STORAGE_CLASS', '')
 
 
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 826b8af11..1631546c4 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -8,7 +8,6 @@
 from io import StringIO
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
 from django.core.management import call_command
 
 from unittest import mock
@@ -43,8 +42,6 @@ def setUp(self):
         self.app = App.objects.all()[0]
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
@@ -129,40 +126,6 @@ def test_config(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         return config5
 
-    def test_default_tags(self, mock_requests):
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = '{"ssd": "true"}'
-        app_id = self.create_app()
-        url = f"/v2/apps/{app_id}/config"
-        response = self.client.get(url)
-        expected = {
-            'owner': self.user.username,
-            'app': app_id,
-            'values': {},
-            'limits': {
-                  PROCFILE_TYPE_RUN: 'std1.large.c1m1',
-                  PROCFILE_TYPE_WEB: 'std1.large.c1m1'
-            },
-            'tags': {'ssd': 'true'},
-            'registry': {}
-        }
-        self.assertEqual(response.data, expected | response.data)
-
-        # make sure changes not drop tags
-        body = {'values': json.dumps({'PORT': '5001'})}
-        response = self.client.post(url, body)
-        expected = {
-            'owner': self.user.username,
-            'app': app_id,
-            'values': {'PORT': '5001'},
-            'limits': {
-                PROCFILE_TYPE_RUN: 'std1.large.c1m1',
-                PROCFILE_TYPE_WEB: 'std1.large.c1m1'
-            },
-            'tags': {'ssd': 'true'},
-            'registry': {}
-        }
-        self.assertEqual(response.data, response.data | expected)
-
     def test_response_data(self, mock_requests):
         """Test that the serialized response contains only relevant data."""
         app_id = self.create_app()
diff --git a/rootfs/api/tests/test_event.py b/rootfs/api/tests/test_event.py
index 93eb3f864..bc02beded 100644
--- a/rootfs/api/tests/test_event.py
+++ b/rootfs/api/tests/test_event.py
@@ -6,7 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
 
 from api.models.app import App
 
@@ -33,8 +32,6 @@ def setUp(self):
         self.app = App.objects.all()[0]
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
index 15b2ab75a..c32cbb6fd 100644
--- a/rootfs/api/tests/test_ptypes.py
+++ b/rootfs/api/tests/test_ptypes.py
@@ -6,7 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
 
 from api.models.app import App
 
@@ -33,8 +32,6 @@ def setUp(self):
         self.app = App.objects.all()[0]
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
diff --git a/rootfs/api/tests/test_resource.py b/rootfs/api/tests/test_resource.py
index 0ddb2551c..3d6590bed 100644
--- a/rootfs/api/tests/test_resource.py
+++ b/rootfs/api/tests/test_resource.py
@@ -6,7 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
 from api.tests import adapter, DryccTransactionTestCase
 from api.exceptions import DryccException
 import requests_mock
@@ -27,8 +26,6 @@ def setUp(self):
         self.app_id = self.create_app()
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
diff --git a/rootfs/api/tests/test_tags.py b/rootfs/api/tests/test_tags.py
index 1913a56f5..b2a9d93ff 100644
--- a/rootfs/api/tests/test_tags.py
+++ b/rootfs/api/tests/test_tags.py
@@ -38,7 +38,7 @@ def test_tags(self, mock_requests):
         self.assertEqual(response.data['tags'], {})
 
         # set some tags
-        body = {'tags': json.dumps({'environ': 'dev'})}
+        body = {'tags': json.dumps({'web': {'environ': 'dev'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         tags1 = response.data
@@ -48,20 +48,19 @@ def test_tags(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('tags', response.data)
         tags = response.data['tags']
-        self.assertIn('environ', tags)
-        self.assertEqual(tags['environ'], 'dev')
+        self.assertIn('web', tags)
+        self.assertEqual(tags['web']['environ'], 'dev')
 
         # set an additional value
-        body = {'tags': json.dumps({'rack': '1'})}
+        body = {'tags': json.dumps({'web': {'rack': '1'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         tags2 = response.data
         self.assertNotEqual(tags1['uuid'], tags2['uuid'])
         tags = response.data['tags']
-        self.assertIn('rack', tags)
-        self.assertEqual(tags['rack'], '1')
-        self.assertIn('environ', tags)
-        self.assertEqual(tags['environ'], 'dev')
+        self.assertIn('web', tags)
+        self.assertEqual(tags['web']['rack'], '1')
+        self.assertEqual(tags['web']['environ'], 'dev')
 
         # read the limit again
         response = self.client.get(url)
@@ -69,13 +68,12 @@ def test_tags(self, mock_requests):
         tags3 = response.data
         self.assertEqual(tags2, tags3)
         tags = response.data['tags']
-        self.assertIn('rack', tags)
-        self.assertEqual(tags['rack'], '1')
-        self.assertIn('environ', tags)
-        self.assertEqual(tags['environ'], 'dev')
+        self.assertIn('web', tags)
+        self.assertEqual(tags['web']['rack'], '1')
+        self.assertEqual(tags['web']['environ'], 'dev')
 
         # unset a value
-        body = {'tags': json.dumps({'rack': None})}
+        body = {'tags': json.dumps({'web': {'rack': None}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         tags4 = response.data
@@ -83,16 +81,16 @@ def test_tags(self, mock_requests):
         self.assertNotIn('rack', json.dumps(response.data['tags']))
 
         # set valid values
-        body = {'tags': json.dumps({'kubernetes.io/hostname': '172.17.8.100'})}
+        body = {'tags': json.dumps({'web': {'kubernetes.io/hostname': '172.17.8.100'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        body = {'tags': json.dumps({'is.valid': 'is-also_valid'})}
+        body = {'tags': json.dumps({'web': {'is.valid': 'is-also_valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        body = {'tags': json.dumps({'host.the-name.com/is.valid': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.the-name.com/is.valid': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        body = {'tags': json.dumps({'host.the-name.com/does.no.exist': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.the-name.com/does.no.exist': 'valid'}})}
         response = self.client.post(url, body)
         self.assertContains(
             response,
@@ -101,23 +99,23 @@ def test_tags(self, mock_requests):
         )
 
         # set invalid values
-        body = {'tags': json.dumps({'valid': 'in\nvalid'})}
+        body = {'tags': json.dumps({'web': {'valid': 'in\nvalid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'host.name.com/notvalid-': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.name.com/notvalid-': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'valid': 'invalid.'})}
+        body = {'tags': json.dumps({'web': {'valid': 'invalid.'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'host.name.com/,not.valid': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.name.com/,not.valid': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
         long_tag = 'a' * 300
-        body = {'tags': json.dumps({'{}/not.valid'.format(long_tag): 'valid'})}
+        body = {'tags': json.dumps({'web': {'{}/not.valid'.format(long_tag): 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'this&foo.com/not.valid': 'valid'})}
+        body = {'tags': json.dumps({'web': {'this&foo.com/not.valid': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index b411bc3dc..1f97239dd 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -6,7 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
 
@@ -26,8 +25,6 @@ def setUp(self):
         self.app_id = self.create_app()
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
diff --git a/rootfs/api/tests/test_workflow_manager.py b/rootfs/api/tests/test_workflow_manager.py
index 44d617444..5d655f86d 100644
--- a/rootfs/api/tests/test_workflow_manager.py
+++ b/rootfs/api/tests/test_workflow_manager.py
@@ -35,8 +35,6 @@ def setUp(self):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index 0d59f5d85..1f9eeef49 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -57,8 +57,6 @@ def manifest(self, namespace, name, image, command, args, spec_annotations, **kw
                 'selector': {
                     'matchLabels': labels
                 },
-                'nodeSelector': kwargs.get('node_selector', {}),
-                'securityContext': kwargs.get('pod_security_context', {}),
             }
         }
 
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 18b5ab9f5..82a3a880a 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -149,7 +149,9 @@ def manifest(self, namespace, name, image, **kwargs):
               'annotations': kwargs.get('annotations', {}),
             },
             'spec': {
-                "affinity": self.affinity(name)
+                "affinity": self.affinity(name),
+                'nodeSelector': kwargs.get('node_selector', {}),
+                'securityContext': kwargs.get('pod_security_context', {}),
             }
         }
         # pod manifest spec
@@ -164,7 +166,7 @@ def manifest(self, namespace, name, image, **kwargs):
         spec['restartPolicy'] = kwargs.get('restart_policy', 'Always')
 
         # apply tags as needed to restrict pod to particular node(s)
-        spec['nodeSelector'] = kwargs.get('tags', {})
+        spec['nodeSelector'].update(kwargs.get('tags', {}))
 
         # How long until a pod is forcefully terminated. 30 is kubernetes default
         spec['terminationGracePeriodSeconds'] = self._get_termination_grace_period(kwargs)  # noqa

From 178a4c658c2a3d0bbbd9f97ef4b7f6bcb0b346ab Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 16 Jul 2024 11:00:55 +0800
Subject: [PATCH 042/200] chore(controller): add node selector status field

---
 rootfs/api/models/app.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 0f0cec4ee..009470653 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -381,6 +381,9 @@ def get_command_and_args(pod, container_name):
                     "lastState": status["lastState"],
                     "ready": status["ready"],
                     "restartCount": status["restartCount"],
+                    "status": pod["status"].get("phase", ""),
+                    "reason": pod["status"].get("reason", ""),
+                    "message": pod["status"].get("message", "")
                 })
         except KubeHTTPException as e:
             if e.response.status_code != 404:
@@ -455,6 +458,7 @@ def describe_deployment(self, deployment_name):
                     "readiness_probe": container.get("readinessProbe", {}),
                     "limits": limits,
                     "volume_mounts": container.get("volumeMounts", []),
+                    "node_selector": deployment["spec"]["template"]['spec'].get("nodeSelector", {})  # noqa
                 })
         except KubeHTTPException as e:
             if e.response.status_code != 404:

From 0b3be32ac20a50b749973d3ddf3a204095b12535 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 17 Jul 2024 16:15:30 +0800
Subject: [PATCH 043/200] chore(filer): add filer response code

---
 rootfs/api/views.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 113790a8b..639ffb3d1 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -867,10 +867,13 @@ def get_client(self):
     def list(self, request, **kwargs):
         path = request.query_params.get('path', '')
         client = self.get_client()
-        results = client.get(path, params={"action": "list"}).json()
-        # fake out pagination for now
-        pagination = {'results': results, 'count': len(results)}
-        return Response(data=pagination)
+        response = client.get(path, params={"action": "list"})
+        if response.status_code == 200:
+            results = response.json()
+            # fake out pagination for now
+            pagination = {'results': results, 'count': len(results)}
+            return Response(data=pagination)
+        return Response(status=response.status_code, data=response.text)
 
     def retrieve(self, request, **kwargs):
         path = self.kwargs.get('path', '')

From a135d06624314fafef04d86a75d1b4e81edfb67f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 18 Jul 2024 13:48:20 +0800
Subject: [PATCH 044/200] feat(release): add run trigger

---
 rootfs/api/models/app.py         |  8 ++---
 rootfs/api/models/release.py     | 15 +++++++++
 rootfs/api/tests/test_release.py | 53 ++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 009470653..84ed5cf1e 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -222,20 +222,20 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
         prefix = f"[pipeline] release {release.version_name}"
         try:
             self.log(f"{prefix} starts running...")
+            procfile_types = release.diff_procfile_types()
             if release.build.dryccfile:
-                if 'run' in release.build.dryccfile:
+                if 'run' in release.build.dryccfile and release.get_run_trigger():
                     self.log(f"{prefix} starts running pipeline.run")
                     job_name = self.run(
                         self.owner, release.get_run_image(), command=release.get_run_command(),
-                        args=release.get_run_args(), timeout=settings.DRYCC_PILELINE_RUN_TIMEOUT,
-                        expires=settings.DRYCC_PILELINE_RUN_TIMEOUT)
+                        args=release.get_run_args(), timeout=release.get_run_timeout(),
+                        expires=release.get_run_timeout())
                     state, labels = 'initializing', {'job-name': job_name}
                     for count, state in enumerate(self.scheduler().pod.watch(
                             self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
                         self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
                     if state != 'down':
                         raise DryccException(f'pipeline run state error: {state}')
-            procfile_types = release.diff_procfile_types()
             if procfile_types is None or len(procfile_types) > 0:
                 self.log(f"{prefix} starts running pipeline.deploy")
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 557486d8b..6c89222bc 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -78,6 +78,21 @@ def get_run_command(self):
         """
         return self.build.dryccfile.get('run', {}).get('command', [])
 
+    def get_run_timeout(self):
+        return int(self.build.dryccfile.get('run', {}).get(
+            'timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT))
+
+    def get_run_trigger(self):
+        if 'ptypes' not in self.build.dryccfile.get('run', {}).get('when', {}):
+            return True
+        procfile_types = self.diff_procfile_types()
+        if procfile_types is None:
+            return True
+        for procfile_type in procfile_types:
+            if procfile_type in self.build.dryccfile['run']['when']['ptypes']:
+                return True
+        return False
+
     def get_deploy_image(self, container_type):
         """
         In the deploy phase of dryccfile
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 32702f771..a16d98bd4 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -892,3 +892,56 @@ def test_diff_procfile_types(self, mock_requests):
                 prefix = f"[{release.app.id}]: [pipeline] release v{release.version}"
                 exp_msg = f"{prefix} no changes, skip executing pipeline.deploy"
                 mock_logger.log.assert_any_call(logging.INFO, exp_msg)
+
+    @override_settings(DRYCC_PILELINE_RUN_TIMEOUT=600)
+    def test_run(self, mock_requests):
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        user = User.objects.get(username='autotest')
+        dryccfile = {
+            "run": {
+                "args": ["sleep", "60s"],
+                "image": "registry.drycc.cc/drycc/base:bookworm"
+            },
+            "deploy": {
+                "web": {
+                    "image": "registry.drycc.cc/drycc/python-dev",
+                    "args": ["python", "-m", "http.server", "5000"]
+                },
+                "task": {
+                    "image": "docker.io/library/nginx:mainline-bookworm-perl",
+                    "command": ["sleep"],
+                    "args": ["infinity"]
+                }
+            }
+        }
+        build = Build.objects.create(
+            owner=user, app=app, image="qwerty", procfile={},
+            sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
+        release = Release.objects.create(
+            version=2, owner=user, app=app, config=app.config_set.latest(),
+            build=build, state="succeed")
+        self.assertEqual(release.get_run_timeout(), 600)
+        self.assertEqual(release.get_run_trigger(), True)
+        # change timeout
+        dryccfile['run']['when'] = {'ptypes': ['web']}
+        dryccfile['run']['timeout'] = 3600
+        dryccfile['deploy']['web']['image'] = 'registry.drycc.cc/drycc/python-dev:latest'
+        build = Build.objects.create(
+            owner=user, app=app, image="qwerty", procfile={},
+            sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
+        release = Release.objects.create(
+            version=3, owner=user, app=app, config=app.config_set.latest(),
+            build=build, state="succeed")
+        self.assertEqual(release.get_run_timeout(), 3600)
+        self.assertEqual(release.get_run_trigger(), True)
+        # change task image
+        dryccfile['deploy']['task']['image'] = 'docker.io/library/nginx:latest'
+        build = Build.objects.create(
+            owner=user, app=app, image="qwerty", procfile={},
+            sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
+        release = Release.objects.create(
+            version=4, owner=user, app=app, config=app.config_set.latest(),
+            build=build, state="succeed")
+        self.assertEqual(release.get_run_timeout(), 3600)
+        self.assertEqual(release.get_run_trigger(), False)

From 3ccb267c1ca7fd438f9595360135b76f8f5e57b6 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 19 Jul 2024 14:02:45 +0800
Subject: [PATCH 045/200] fix(filer): app permission error

---
 rootfs/api/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 639ffb3d1..23e43d2a5 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -854,14 +854,14 @@ def path(self, request, *args, **kwargs):
         return Response(serializer.data)
 
 
-class AppFilerClientViewSet(BaseDryccViewSet):
+class AppFilerClientViewSet(AppResourceViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
     parser_classes = [MultiPartParser]
 
     def get_client(self):
         volume = get_object_or_404(
-            models.volume.Volume, app__id=self.kwargs['id'], name=self.kwargs['name'])
+            models.volume.Volume, app=self.get_app(), name=self.kwargs['name'])
         return filer.FilerClient(volume.app.id, volume, volume.app.scheduler())
 
     def list(self, request, **kwargs):

From 6b8d9f2db250cf6159c5697462bb6094edc599f2 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 19 Jul 2024 14:48:41 +0800
Subject: [PATCH 046/200] fix(controller): pods ptypes request app permission
 error

---
 rootfs/api/views.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 23e43d2a5..96a3cc1c8 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -402,12 +402,12 @@ def post_save(self, config):
             raise DryccException(str(e)) from e
 
 
-class PodViewSet(BaseDryccViewSet):
+class PodViewSet(AppResourceViewSet):
     model = models.app.App
     serializer_class = serializers.PodSerializer
 
     def list(self, *args, **kwargs):
-        pods = self.get_object().list_pods(*args, **kwargs)
+        pods = self.get_app().list_pods(*args, **kwargs)
         data = self.get_serializer(pods, many=True).data
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
@@ -415,7 +415,7 @@ def list(self, *args, **kwargs):
 
     def describe(self, *args, **kwargs):
         pod_name = kwargs["name"]
-        data = self.get_object().describe_pod(pod_name)
+        data = self.get_app().describe_pod(pod_name)
         if len(data) == 0:
             raise DryccException("this process not found")
         # fake out pagination for now
@@ -426,19 +426,16 @@ def delete(self, request, **kwargs):
         pod_names = request.data.get("pod_ids")
         pod_names = pod_names.split(",")
         for pod_name in set(pod_names):
-            delete_pod.delay(self.get_object(), **{"pod_name": pod_name})
+            delete_pod.delay(self.get_app(), **{"pod_name": pod_name})
         return Response(status=status.HTTP_200_OK)
 
 
-class PtypesViewSet(BaseDryccViewSet):
+class PtypesViewSet(AppResourceViewSet):
     model = models.app.App
     serializer_class = serializers.PtypesSerializer
 
-    def get_queryset(self, *args, **kwargs):
-        return self.model.objects.all(*args, **kwargs)
-
     def list(self, *args, **kwargs):
-        deploys = self.get_object().list_deployments(*args, **kwargs)
+        deploys = self.get_app().list_deployments(*args, **kwargs)
         data = self.get_serializer(deploys, many=True).data
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
@@ -446,7 +443,7 @@ def list(self, *args, **kwargs):
 
     def describe(self, *args, **kwargs):
         deployment_name = kwargs["name"]
-        data = self.get_object().describe_deployment(deployment_name)
+        data = self.get_app().describe_deployment(deployment_name)
         if len(data) == 0:
             raise DryccException("this ptype not found")
         # fake out pagination for now
@@ -454,7 +451,7 @@ def describe(self, *args, **kwargs):
         return Response(pagination, status=status.HTTP_200_OK)
 
     def restart(self, request, *args, **kwargs):
-        app = self.get_object()
+        app = self.get_app()
         ptypes = []
         types = request.data.get("types", "").split(",")
         types = [ptype for ptype in set(types) if ptype != ""]
@@ -472,7 +469,7 @@ def restart(self, request, *args, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def scale(self, request, **kwargs):
-        app = self.get_object()
+        app = self.get_app()
         latest_release = app.release_set.filter(failed=False).latest()
         if latest_release.build is not None and latest_release.state == "created":
             raise DryccException('There is an executing pipeline, please wait')

From 55b5acfac29194b1bd6f5f4610cf1bc68638f041 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sun, 21 Jul 2024 02:24:59 +0800
Subject: [PATCH 047/200] fix(charts): miss mutate ca cert

---
 .../templates/controller-mutate-webhook.yaml  | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/charts/controller/templates/controller-mutate-webhook.yaml b/charts/controller/templates/controller-mutate-webhook.yaml
index 637bd3ac5..7da8c87e0 100644
--- a/charts/controller/templates/controller-mutate-webhook.yaml
+++ b/charts/controller/templates/controller-mutate-webhook.yaml
@@ -7,16 +7,20 @@ apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
   name: {{ .Release.Namespace }}-controller-mutate
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
+  labels:
+    heritage: drycc
 webhooks:
 - name: mutate.drycc.cc
   sideEffects: None
   admissionReviewVersions: ["v1"]
   clientConfig:
-    caBundle: {{ b64enc $ca.Cert }}
+    caBundle: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "ca.crt" "defaultValue" $ca.Cert "context" $)) }}
     service:
       name: drycc-controller-mutate
       namespace: "{{ .Release.Namespace }}"
-      path: "{{ printf "/v2/mutate/%s/" $key }}"
+      path: "{{ printf "/v2/mutate/%s/" ((include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "key" "defaultValue" $key "context" $)) | b64dec) }}"
   failurePolicy: Fail
   objectSelector:
     matchLabels:
@@ -40,11 +44,14 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: controller-mutate-cert
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
   labels:
-    release: "{{ .Release.Name }}"
-    heritage: "{{ .Release.Service }}"
+    heritage: drycc
 type: Opaque
 data:
-  key: {{ b64enc $key }}
-  tls.crt: {{ b64enc $cert.Cert }}
-  tls.key: {{ b64enc $cert.Key }}
+  key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "key" "defaultValue" $key "context" $)) }}
+  ca.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "ca.key" "defaultValue" $ca.Key "context" $)) }}
+  ca.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "ca.crt" "defaultValue" $ca.Cert "context" $)) }}
+  tls.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "tls.key" "defaultValue" $cert.Key "context" $)) }}
+  tls.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "tls.crt" "defaultValue" $cert.Cert "context" $)) }}

From 547f010d019c7ba0c5134ceb944368551ccfc5e9 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Mon, 22 Jul 2024 10:48:50 +0800
Subject: [PATCH 048/200] fix(charts): lookup mutate ca cert (#146)

---
 .../templates/controller-mutate-webhook.yaml           | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/charts/controller/templates/controller-mutate-webhook.yaml b/charts/controller/templates/controller-mutate-webhook.yaml
index 7da8c87e0..bc7eb614b 100644
--- a/charts/controller/templates/controller-mutate-webhook.yaml
+++ b/charts/controller/templates/controller-mutate-webhook.yaml
@@ -50,8 +50,8 @@ metadata:
     heritage: drycc
 type: Opaque
 data:
-  key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "key" "defaultValue" $key "context" $)) }}
-  ca.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "ca.key" "defaultValue" $ca.Key "context" $)) }}
-  ca.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "ca.crt" "defaultValue" $ca.Cert "context" $)) }}
-  tls.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "tls.key" "defaultValue" $cert.Key "context" $)) }}
-  tls.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "tls.crt" "defaultValue" $cert.Cert "context" $)) }}
+  key: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "key" "defaultValue" $key "context" $)) }}
+  ca.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "ca.key" "defaultValue" $ca.Key "context" $)) }}
+  ca.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "ca.crt" "defaultValue" $ca.Cert "context" $)) }}
+  tls.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "tls.key" "defaultValue" $cert.Key "context" $)) }}
+  tls.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "tls.crt" "defaultValue" $cert.Cert "context" $)) }}

From d76dfdd521e6f4deff1fa51c613c3ca30a7e53e3 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 23 Jul 2024 16:55:58 +0800
Subject: [PATCH 049/200] chore(charts): config.defaultMetrics add field (#147)

---
 charts/controller/templates/_helpers.tpl | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 31b65bc92..2ff44e7d0 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -279,7 +279,7 @@ resources:
     limits:
       cpu: {{ $cpu }}
       memory: {{ $memory }}Gi
-      ephemeral-storage: 2Gi
+      ephemeral-storage: 4Gi
     requests:
       cpu: {{ divf $cpu 4 }}
       memory: {{ divf $memory 2 }}Gi
@@ -366,4 +366,11 @@ kube_pod_container_status_ready: [container, namespace, node, pod]
 kube_pod_container_status_terminated: [container, namespace, node, pod]
 kube_pod_container_status_waiting: [container, namespace, node, pod]
 kube_pod_container_status_restarts_total: [container, namespace, node, pod]
+kube_pod_spec_volumes_persistentvolumeclaims_info: [namespace, pod, persistentvolumeclaim]
+kubelet_volume_stats_used_bytes: [namespace, persistentvolumeclaim, job]
+kubelet_volume_stats_available_bytes: [namespace, persistentvolumeclaim, job]
+kubelet_volume_stats_capacity_bytes: [namespace, persistentvolumeclaim, job]
+kubelet_volume_stats_inodes: [namespace, persistentvolumeclaim, job]
+kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim, job]
+kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim, job]
 {{- end }}
\ No newline at end of file

From a38770838f8946575bc4fb247f28aed6175e58f2 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 6 Aug 2024 10:37:21 +0800
Subject: [PATCH 050/200] chore(tests): use latest codecov

---
 Makefile                    | 2 +-
 rootfs/Dockerfile.test      | 2 ++
 rootfs/bin/test-unit        | 1 -
 rootfs/bin/upload-coverage  | 6 +++++-
 rootfs/dev_requirements.txt | 9 +++------
 5 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 62c6b6ac7..da92b03f1 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,6 @@ test-integration:
 
 upload-coverage:
 	$(eval CI_ENV := $(shell curl -s https://codecov.io/env | bash))
-	podman run --rm ${CI_ENV} -v ${CURDIR}:/test -w /test/rootfs ${IMAGE}.test /test/rootfs/bin/upload-coverage
+	podman run --rm ${CI_ENV} -v ${CURDIR}:/test -w /test/rootfs -e CODECOV_TOKEN=${CODECOV_TOKEN} ${IMAGE}.test /test/rootfs/bin/upload-coverage
 
 .PHONY: check-kubectl check-podman build podman-build podman-build-test deploy clean commit-hook full-clean test test-style test-unit test-functional test-integration upload-coverage
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index 1aa3698f6..9d43c0b03 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -15,6 +15,8 @@ ENV PGDATA="/opt/drycc/postgresql/data" \
 
 RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
   install-packages mercurial ca-certificates git inotify-tools $buildDeps \
+  && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \
+  && chmod +x /usr/local/bin/codecov \
   && install-stack python $PYTHON_VERSION \
   && install-stack redis $REDIS_VERSION \
   && install-stack rabbitmq $RABBITMQ_VERSION \
diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit
index 3afb77c1d..54d2ba37c 100755
--- a/rootfs/bin/test-unit
+++ b/rootfs/bin/test-unit
@@ -23,4 +23,3 @@ start_redis
 start_rabbitmq
 python3 manage.py check
 coverage run --source=api,scheduler manage.py test --settings=api.settings.testing --noinput api scheduler.tests --exclude-tag=auth
-coverage report -m
diff --git a/rootfs/bin/upload-coverage b/rootfs/bin/upload-coverage
index 53433f3d0..492452b18 100755
--- a/rootfs/bin/upload-coverage
+++ b/rootfs/bin/upload-coverage
@@ -6,4 +6,8 @@
 # fail hard and fast even on pipelines
 set -eou pipefail
 
-codecov --required
+coverage report -m > coverage.txt
+
+if [[ -n $CODECOV_TOKEN ]]; then
+  codecov upload-process --plugin=noop -t "$CODECOV_TOKEN"
+fi
diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt
index 5e6b68009..ca19f67a2 100644
--- a/rootfs/dev_requirements.txt
+++ b/rootfs/dev_requirements.txt
@@ -1,13 +1,10 @@
 # test module
 # test
 # Run "make test-unit" for the % of code exercised during tests
-coverage==7.4.4
+coverage==7.6.1
 
 # Run "make test-style" to check python syntax and style
-flake8==7.0.0
-
-# code coverage report at https://codecov.io/github/drycc/controller
-codecov==2.1.13
+flake8==7.1.1
 
 # mock out python-requests, mostly k8s
-requests-mock==1.11.0
+requests-mock==1.12.1

From 6164fe96ed5b223565dcbcc67545ff3b35e36e26 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 7 Aug 2024 18:11:55 +0800
Subject: [PATCH 051/200] chore(charts): modify grpcroutes role (#148)

---
 charts/controller/templates/controller-clusterrole.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index 34f521743..0084190cd 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -80,7 +80,7 @@ rules:
   resources: ["serviceinstances", "servicebindings"]
   verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
 - apiGroups: ["gateway.networking.k8s.io"]
-  resources: ["gateways", "httproutes", "grocroutes", "tcproutes", "udproutes", "tlsroutes"]
+  resources: ["gateways", "httproutes", "grpcroutes", "tcproutes", "udproutes", "tlsroutes"]
   verbs: ["get", "patch", "list", "create", "update", "delete"]
 - apiGroups: ["batch"]
   resources: ["jobs"]

From 467ef80806fc42560a89b8e918bf482b22a6c946 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 8 Aug 2024 13:55:07 +0800
Subject: [PATCH 052/200] chore(deps): bump django from 4.2.14 to 4.2.15 in
 /rootfs (#149)

---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 44cb0260e..6a79ceb15 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.14
+django==4.2.15
 channels==4.1.0
 aiohttp==3.9.4
 django-cors-headers==4.3.1

From 3bb209a52ebb97a5fe3bac3bfc65a549e9853d41 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 9 Aug 2024 14:39:39 +0800
Subject: [PATCH 053/200] chore(metrics): add prometheus query interval

---
 rootfs/api/monitor.py             | 4 +++-
 rootfs/api/settings/production.py | 1 +
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index e56431095..4d2bb4edd 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -119,7 +119,9 @@ async def last_metrics(namespace):
     if not settings.DRYCC_METRICS_CONFIG:
         return
     promql = query_last_metrics_promql_tpl % (
-        '|'.join(settings.DRYCC_METRICS_CONFIG.keys()), namespace, '1m')
+      '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
+      namespace,
+      settings.DRYCC_METRICS_INTERVAL)
     url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
     params = {"query": promql, "start": int(time.time() - 60)}
     async with aiohttp.ClientSession() as session:
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index ce173f110..83c27d57d 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -281,6 +281,7 @@
     with open(DRYCC_METRICS_CONFIG_PATH) as fd:
         DRYCC_METRICS_CONFIG = json.load(fd)
 DRYCC_METRICS_EXPIRY = int(os.environ.get('DRYCC_METRICS_EXPIRY', '20'))
+DRYCC_METRICS_INTERVAL = os.environ.get('DRYCC_METRICS_INTERVAL', '5m')
 
 # security keys and auth tokens
 random_secret = 'CHANGEME_sapm$s%upvsw5l_zuy_&29rkywd^78ff(qi*#@&*^'

From fbd0ab049839a7edddb76b3cc79995cf4ba5fe98 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 Aug 2024 17:53:49 +0800
Subject: [PATCH 054/200] chore(deps): bump aiohttp from 3.9.4 to 3.10.2 in
 /rootfs (#152)

---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 6a79ceb15..dac0e9403 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -3,7 +3,7 @@ inotify==0.2.10
 backoff==2.2.1
 django==4.2.15
 channels==4.1.0
-aiohttp==3.9.4
+aiohttp==3.10.2
 django-cors-headers==4.3.1
 django-guardian==2.4.0
 djangorestframework==3.15.2

From 5a9f5af79d2fdf864a8754345907b0d6b6ce24bd Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 13 Aug 2024 10:26:53 +0800
Subject: [PATCH 055/200] chore(perms): add object perms

---
 rootfs/api/consumers.py                       |   4 +-
 .../0010_alter_certificate_options.py         |  17 ++
 rootfs/api/models/app.py                      |  10 +-
 rootfs/api/models/base.py                     |  33 ++++
 rootfs/api/models/certificate.py              |   8 +-
 rootfs/api/permissions.py                     |  45 +++---
 rootfs/api/tests/test_app.py                  |  10 +-
 rootfs/api/tests/test_hooks.py                |  10 +-
 rootfs/api/tests/test_perm.py                 | 113 +++++++++----
 rootfs/api/urls.py                            |  20 +--
 rootfs/api/views.py                           | 153 +++++++++---------
 11 files changed, 269 insertions(+), 154 deletions(-)
 create mode 100644 rootfs/api/migrations/0010_alter_certificate_options.py

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index e1552043d..a04186050 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -20,7 +20,7 @@
 from channels.generic.websocket import AsyncWebsocketConsumer
 
 from .models.app import App
-from .permissions import has_app_permission
+from .permissions import has_object_permission
 
 
 class BaseAppConsumer(AsyncWebsocketConsumer):
@@ -35,7 +35,7 @@ def has_perm(self):
         if permission is None:
             try:
                 app = App.objects.get(id=self.id)
-                permission = has_app_permission(self.scope["user"], app, "GET")
+                permission = has_object_permission(self.scope["user"], app, "GET")
                 if permission[0]:
                     cache.set(key, permission, timeout=self.timeout)
             except App.DoesNotExist:
diff --git a/rootfs/api/migrations/0010_alter_certificate_options.py b/rootfs/api/migrations/0010_alter_certificate_options.py
new file mode 100644
index 000000000..fd5e60f4b
--- /dev/null
+++ b/rootfs/api/migrations/0010_alter_certificate_options.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2.15 on 2024-08-14 09:43
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0009_remove_appsettings_canaries_remove_release_canary_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='certificate',
+            options={'ordering': ['name', 'common_name', 'expires'], 'permissions': (('use_cert', 'Can use cert'),)},
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 84ed5cf1e..515ce2d33 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -31,10 +31,12 @@
 from .tls import TLS
 from .appsettings import AppSettings
 from .volume import Volume
-from .base import UuidAuditedModel, PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN, DEFAULT_HTTP_PORT
+from .base import UuidAuditedModel, PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN, DEFAULT_HTTP_PORT, \
+    ObjectPolicy, object_policy_registry
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
+app_policy = ObjectPolicy('id', 'use_app', 'Can use app')  # query field, policy, description
 
 
 # http://kubernetes.io/v1.1/docs/design/identifiers.html
@@ -77,7 +79,7 @@ class App(UuidAuditedModel):
 
     class Meta:
         verbose_name = 'Application'
-        permissions = (('use_app', 'Can use app'),)
+        permissions = (app_policy[1:], )
         ordering = ['id']
 
     def save(self, *args, **kwargs):
@@ -1161,3 +1163,7 @@ def _gather_app_settings(self, release, app_settings, procfile_type, replicas, v
             'pod_security_context': limit_plan.pod_security_context,
             'container_security_context': limit_plan.container_security_context,
         }
+
+
+# Register policy
+object_policy_registry.register(App, app_policy)
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 87ae5353a..038de2970 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -4,6 +4,7 @@
 import importlib
 from datetime import timedelta
 from functools import partial
+from collections import namedtuple
 from django.db import models
 from django.conf import settings
 from django.utils import timezone
@@ -38,6 +39,38 @@
 def get_anonymous_user_instance(user): return user(id=-1, username=settings.ANONYMOUS_USER_NAME)
 
 
+ObjectPolicy = namedtuple('ObjectPolicy', ['unique', 'codename', 'description'])
+
+
+class ObjectPolicyRegistry(object):
+
+    def __init__(self):
+        self.object_permission_policy_table = dict[type, ObjectPolicy]()
+
+    def all(self) -> list[type, ObjectPolicy]:
+        return self.object_permission_policy_table.items()
+
+    def get(self, query) -> tuple[type, ObjectPolicy]:
+        if isinstance(query, str):
+            for key, value in self.object_permission_policy_table.items():
+                if value.codename == query:
+                    return key, value
+        elif isinstance(query, models.Model):
+            query = type(query)
+            value = self.object_permission_policy_table.get(query)
+            return query if value else None, value
+        elif isinstance(query, type):
+            value = self.object_permission_policy_table.get(query)
+            return query if value else None, value
+        return None, None
+
+    def register(self, cls: type, object_policy: ObjectPolicy) -> None:
+        self.object_permission_policy_table[cls] = object_policy
+
+
+object_policy_registry = ObjectPolicyRegistry()
+
+
 class AuditedModel(models.Model):
     """Add created and updated fields to a model."""
 
diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index 45c473a99..7c002d3e3 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -15,11 +15,12 @@
 from api.utils import validate_label
 from api.exceptions import AlreadyExists, ServiceUnavailable
 from scheduler import KubeException
-from .base import AuditedModel
+from .base import AuditedModel, ObjectPolicy, object_policy_registry
 from .domain import Domain
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
+cert_policy = ObjectPolicy('name', 'use_cert', 'Can use cert')
 
 
 # Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
@@ -120,6 +121,7 @@ class Certificate(AuditedModel):
     subject = models.TextField(editable=False)
 
     class Meta:
+        permissions = (cert_policy[1:], )
         ordering = ['name', 'common_name', 'expires']
 
     @property
@@ -238,3 +240,7 @@ def detach(self, *args, **kwargs):
                 raise ServiceUnavailable(
                     "Could not delete certificate secret {} for application {}".format(
                         self.certname, namespace)) from e
+
+
+# Register policy
+object_policy_registry.register(Certificate, cert_policy)
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index f7fcab261..8ea9444ea 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -3,13 +3,15 @@
 from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
 from api import manager
-from api.models.blocklist import Blocklist, App
+from api.models import app
+from api.models import blocklist
+from api.models.base import object_policy_registry
 
 
 def get_app_status(app):
-    blocklist = Blocklist.get_blocklist(app)
-    if blocklist:
-        return False, blocklist.remark
+    block = blocklist.Blocklist.get_blocklist(app)
+    if block:
+        return False, block.remark
     if settings.WORKFLOW_MANAGER_URL:
         status = manager.User().get_status(app.owner.pk)
         if not status["is_active"]:
@@ -17,23 +19,22 @@ def get_app_status(app):
     return True, None
 
 
-def has_app_permission(user, obj, method):
-    if isinstance(obj, App) or hasattr(obj, 'app'):
-        app = obj if isinstance(obj, App) else obj.app
-        is_ok, message = get_app_status(app)
-        if is_ok:
-            if user.is_superuser:
-                return True, None
-            elif app.owner == user:
-                return True, None
-            elif user.is_staff or user.has_perm('use_app', app):
-                if method != 'DELETE':
-                    return True, None
-                else:
-                    return False, "User does not have permission to delete"
+def has_object_permission(user, obj, method):
+    obj = getattr(obj, 'app', obj)
+    object_policy = object_policy_registry.get(obj)[1]
+    has_permission, message = False, f"{obj} object does not exist or does not have permission."
+    if user.is_superuser:
+        has_permission, message = True, None
+    elif getattr(obj, "owner", None) == user:
+        has_permission, message = True, None
+    elif user.is_staff or (object_policy and user.has_perm(object_policy.codename, obj)):
+        if method != 'DELETE':
+            has_permission, message = True, None
         else:
-            return is_ok, message
-    return False, "App object does not exist or does not have permission."
+            has_permission, message = False, "{user} does not have permission to delete."
+    elif has_permission and isinstance(obj, app.App):
+        return get_app_status(obj)
+    return has_permission, message
 
 
 class IsAnonymous(permissions.BasePermission):
@@ -75,13 +76,13 @@ def has_object_permission(self, request, view, obj):
             return False
 
 
-class IsAppUser(permissions.BasePermission):
+class IsObjectUser(permissions.BasePermission):
     """
     Object-level permission to allow owners or collaborators to access
     an app-related model.
     """
     def has_object_permission(self, request, view, obj):
-        return has_app_permission(request.user, obj, request.method)[0]
+        return has_object_permission(request.user, obj, request.method)[0]
 
 
 class IsAdmin(permissions.BasePermission):
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 80adf01dc..25b81da22 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -22,6 +22,7 @@
 
 from api.exceptions import DryccException
 from api.tests import adapter, DryccTestCase
+from api.models.base import object_policy_registry
 import requests_mock
 
 User = get_user_model()
@@ -358,9 +359,12 @@ def test_app_transfer(self, mock_requests):
         self.assertEqual(config2.owner, collaborator)
 
         # Collaborators can't transfer
-        body = {'username': owner.username}
-        perms_url = url+"/perms/"
-        response = self.client.post(perms_url, body)
+        body = {
+            'username': owner.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app.id,
+        }
+        response = self.client.post('/v2/perms/rules/', body)
         self.assertEqual(response.status_code, 201, response.data)
 
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index 432c46a99..81923dd2d 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -8,6 +8,8 @@
 from django.core.cache import cache
 
 from api.tests import adapter, DryccTransactionTestCase
+from api.models.app import App
+from api.models.base import object_policy_registry
 import requests_mock
 
 User = get_user_model()
@@ -70,8 +72,12 @@ def test_key_hook(self, mock_requests):
         app_id = self.create_app()
 
         # give user permission to app
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': str(self.user)}
+        body = {
+            'username': str(self.user),
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
+        url = "/v2/perms/rules"
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
diff --git a/rootfs/api/tests/test_perm.py b/rootfs/api/tests/test_perm.py
index 2969859c0..80cf8206f 100644
--- a/rootfs/api/tests/test_perm.py
+++ b/rootfs/api/tests/test_perm.py
@@ -1,11 +1,13 @@
 from django.test import tag
 from django.contrib.auth import get_user_model
 from api.tests import DryccTestCase
+from api.models.base import object_policy_registry
+from api.models.app import App
 
 User = get_user_model()
 
 
-class TestAppPerms(DryccTestCase):
+class TestUserPerm(DryccTestCase):
 
     fixtures = ['test_sharing.json']
 
@@ -21,6 +23,18 @@ def setUp(self):
         self.user3 = User.objects.get(username='autotest-3')
         self.token3 = self.get_or_create_token(self.user3)
 
+    @tag('auth')
+    def test_codes(self):
+        response = self.client.get('/v2/perms/codes/')
+        expect = {
+            'results': [
+                {'codename': 'use_app', 'description': 'Can use app'},
+                {'codename': 'use_cert', 'description': 'Can use cert'}
+            ],
+            'count': 2
+        }
+        self.assertEqual(expect, response.json())
+
     @tag('auth')
     def test_create(self):
         # check that user 1 sees her lone app and user 2's app
@@ -43,13 +57,19 @@ def test_create(self):
             self.assertEqual(response.data['detail'],
                              'You do not have permission to perform this action.', msg=msg)
 
-        # TODO: test that git pushing to the app fails
-        # give user 2 permission to user 1's app
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
+        url = "/v2/perms/rules/"
+        for cls, object_policy in object_policy_registry.all():
+            if not cls.objects.exists():
+                continue
+            for obj in cls.objects.filter(owner=self.user):
+                body = {
+                    'username': self.user2.username,
+                    'codename': object_policy.codename,
+                    'uniqueid': getattr(obj, object_policy.unique)
+                }
+                self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+                response = self.client.post(url, body)
+                self.assertEqual(response.status_code, 201, response.data)
 
         # check that user 2 can see the app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
@@ -71,11 +91,15 @@ def test_create_errors(self):
         response = self.client.get('/v2/apps')
         app_id = response.data['results'][0]['id']
 
+        body = {
+            'username': self.user2.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
+
         # check that user 2 can't create a permission
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post(url, body)
+        response = self.client.post("/v2/perms/rules/", body)
         self.assertEqual(response.status_code, 403)
 
     @tag('auth')
@@ -83,9 +107,12 @@ def test_delete(self):
         # give user 2 permission to user 1's app
         response = self.client.get('/v2/apps')
         app_id = response.data['results'][0]['id']
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
-        response = self.client.post(url, body)
+        body = {
+            'username': self.user2.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
+        response = self.client.post("/v2/perms/rules/", body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # check that user 2 can see the app as well as his own
@@ -96,7 +123,8 @@ def test_delete(self):
 
         # delete permission to user 1's app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        url = "/v2/apps/{}/perms/{}".format(app_id, 'autotest-2')
+        response = self.client.get("/v2/perms/rules/")
+        url = "/v2/perms/rules/{}/".format(response.json()["results"][0]["id"])
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
         self.assertIsNone(response.data)
@@ -109,7 +137,7 @@ def test_delete(self):
         # delete permission to user 1's app again, expecting an error
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
     @tag('auth')
     def test_list(self):
@@ -120,14 +148,17 @@ def test_list(self):
         app_id = response.data['results'][0]['id']
 
         # create a new object permission
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
-        response = self.client.post(url, body)
+        body = {
+            'username': self.user2.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
+        response = self.client.post("/v2/perms/rules/", body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # list perms on the app
-        response = self.client.get("/v2/apps/{}/perms".format(app_id))
-        self.assertEqual(response.data, {'users': ['autotest-2']})
+        response = self.client.get("/v2/perms/rules/")
+        self.assertEqual(response.data["count"], 1)
 
     @tag('auth')
     def test_admin_can_list(self):
@@ -139,12 +170,11 @@ def test_admin_can_list(self):
     @tag('auth')
     def test_list_errors(self):
         response = self.client.get('/v2/apps')
-        app_id = response.data['results'][0]['id']
-
         # login as user 2, list perms on the app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get("/v2/apps/{}/perms".format(app_id))
-        self.assertEqual(response.status_code, 403)
+        response = self.client.get("/v2/perms/rules/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data, {'results': [], 'count': 0})
 
     @tag('auth')
     def test_unauthorized_user_cannot_modify_perms(self):
@@ -159,10 +189,13 @@ def test_unauthorized_user_cannot_modify_perms(self):
         body = {'id': app_id}
         response = self.client.post(url, body)
 
-        url = '{}/{}/perms'.format(url, app_id)
-        body = {'username': self.user2.username}
+        body = {
+            'username': self.user2.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post(url, body)
+        response = self.client.post('/v2/perms/rules/', body)
         self.assertEqual(response.status_code, 403)
 
     @tag('auth')
@@ -174,17 +207,25 @@ def test_collaborator_cannot_share(self):
         owner_token = self.token
         collab = self.user2
         collab_token = self.token2
-        url = '/v2/apps/{}/perms'.format(app_id)
 
         # Share app with collaborator
-        body = {'username': collab.username}
+        body = {
+            'username': collab.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
+        url = '/v2/perms/rules/'
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # Collaborator should fail to share app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
-        body = {'username': self.user3.username}
+        body = {
+            'username': self.user3.username,
+            'codename': object_policy_registry.get(App)[1].codename,
+            'uniqueid': app_id,
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
 
@@ -197,16 +238,18 @@ def test_collaborator_cannot_share(self):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token3)
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['count'], 1, response.data)
 
         # Collaborator cannot delete other collaborator
-        url += "/{}".format(self.user3.username)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
+        url += "{}/".format(response.data["results"][0]["id"])
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 403)
 
         # Collaborator can delete themselves
-        url = '/v2/apps/{}/perms/{}'.format(app_id, collab.username)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token3)
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204, response.data)
+        self.assertEqual(response.status_code, 204)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 69fd4b291..312aecf74 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -97,13 +97,6 @@
     re_path(
         r"^apps/(?P<id>{})/tls/?$".format(settings.APP_URL_REGEX),
         views.TLSViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
-    # apps sharing
-    re_path(
-        r"^apps/(?P<id>{})/perms/(?P<username>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
-        views.AppPermsViewSet.as_view({'delete': 'destroy'})),
-    re_path(
-        r"^apps/(?P<id>{})/perms/?$".format(settings.APP_URL_REGEX),
-        views.AppPermsViewSet.as_view({'get': 'list', 'post': 'create'})),
     # application volumes
     re_path(
         r"^apps/(?P<id>{})/volumes/?$".format(settings.APP_URL_REGEX),
@@ -173,13 +166,6 @@
     re_path(
         r'^auth/whoami/?$',
         views.UserManagementViewSet.as_view({'get': 'list'})),
-    # admin sharing
-    re_path(
-        r'^admin/perms/(?P<username>[\w.@+-]+)/?$',
-        views.AdminPermsViewSet.as_view({'delete': 'destroy'})),
-    re_path(
-        r'^admin/perms/?$',
-        views.AdminPermsViewSet.as_view({'get': 'list', 'post': 'create'})),
     # certificates
     re_path(
         r'^certs/(?P<name>[-_*.\w]+)/domain/(?P<domain>\**\.?[-\._\w]+)?/?$',
@@ -233,6 +219,12 @@
     re_path(
         r'^manager/(?P<type>[\w.@+-]+)s/(?P<id>{})/unblock/?$'.format(settings.APP_URL_REGEX),
         views.WorkflowManagerViewset.as_view({'delete': 'unblock'})),
+    # user perms
+    re_path(r"^perms/codes/?$", views.UserPermViewSet.as_view({'get': 'codes'})),
+    re_path(
+        r"^perms/rules/?$", views.UserPermViewSet.as_view({'get': 'list', 'post': 'create'})),
+    re_path(
+        r"^perms/rules/(?P<id>[-_\w]+)/?$", views.UserPermViewSet.as_view({'delete': 'destroy'})),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
     re_path(r"^tokens/(?P<pk>[-_\w]+)/?$", views.TokenViewSet.as_view({'delete': 'destroy'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 96a3cc1c8..5bebce5ce 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -13,13 +13,12 @@
 from django.contrib.auth import get_user_model
 from django.shortcuts import get_object_or_404, redirect
 from guardian.shortcuts import assign_perm, get_objects_for_user, \
-    get_users_with_perms, remove_perm
+    get_users_with_perms, remove_perm, UserObjectPermission
 from django.views.generic import View
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
 from django.views.decorators.vary import vary_on_headers
 from rest_framework import renderers, status, filters
-from rest_framework.exceptions import PermissionDenied, NotFound
 from rest_framework.permissions import IsAuthenticated, AllowAny
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
@@ -244,7 +243,7 @@ class BaseDryccViewSet(viewsets.OwnerViewSet):
     the `model` attribute shortcut.
     """
     lookup_field = 'id'
-    permission_classes = [IsAuthenticated, permissions.IsAppUser]
+    permission_classes = [IsAuthenticated, permissions.IsObjectUser]
     renderer_classes = [renderers.JSONRenderer]
 
 
@@ -295,7 +294,7 @@ def list(self, request, *args, **kwargs):
         interact with.
         """
         queryset = super(AppViewSet, self).get_queryset(**kwargs) | \
-            get_objects_for_user(self.request.user, 'api.use_app')
+            get_objects_for_user(self.request.user, f'api.{models.app.app_policy[1]}')
         instance = self.filter_queryset(queryset)
         page = self.paginate_queryset(instance)
         if page is not None:
@@ -327,7 +326,7 @@ def update(self, request, **kwargs):
 
         if request.data.get('owner'):
             if self.request.user != app.owner and not self.request.user.is_superuser:
-                raise PermissionDenied()
+                return Response(status=status.HTTP_403_FORBIDDEN)
             new_owner = get_object_or_404(User, username=request.data['owner'])
             # ensure all downstream objects that are owned by this user and are part of this app
             # is also updated
@@ -575,6 +574,11 @@ class CertificateViewSet(BaseDryccViewSet):
     model = models.certificate.Certificate
     serializer_class = serializers.CertificateSerializer
 
+    def get_queryset(self, **kwargs):
+        return self.model.objects.filter(owner=self.request.user, **kwargs) | \
+            get_objects_for_user(
+                self.request.user, f'api.{models.certificate.cert_policy[1]}')
+
     def get_object(self, **kwargs):
         """Retrieve domain certificate by its name"""
         qs = self.get_queryset(**kwargs)
@@ -598,7 +602,13 @@ def detach(self, request, *args, **kwargs):
             self.get_object().detach(*args, **kwargs)
         except Http404:
             raise
+        return Response(status=status.HTTP_204_NO_CONTENT)
 
+    def destroy(self, *args, **kwargs):
+        resource = self.get_object()
+        if self.request.user != resource.owner and not self.request.user.is_superuser:
+            return Response(status=status.HTTP_403_FORBIDDEN)
+        resource.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
@@ -653,7 +663,7 @@ def public_key(self, request, *args, **kwargs):
         key = get_object_or_404(models.key.Key, fingerprint=fingerprint)
 
         queryset = models.app.App.objects.all() | \
-            get_objects_for_user(self.request.user, 'api.use_app')
+            get_objects_for_user(self.request.user, f'api.{models.app.app_policy[1]}')
         items = self.filter_queryset(queryset)
 
         apps = []
@@ -669,10 +679,8 @@ def public_key(self, request, *args, **kwargs):
 
     def app(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
-
-        perm_name = "api.use_app"
         usernames = [u.id for u in get_users_with_perms(app)
-                     if u.has_perm(perm_name, app)]
+                     if u.has_perm(f"api.{models.app.app_policy[1]}", app)]
 
         data = {}
         result = models.key.Key.objects \
@@ -695,10 +703,10 @@ def users(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
         request.user = get_object_or_404(User, username=kwargs['username'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(
+        has_permission, message = permissions.has_object_permission(
             request.user, app, request.method)
         if not has_permission:
-            raise PermissionDenied(message)
+            return Response(message, status=status.HTTP_403_FORBIDDEN)
 
         data = {request.user.username: []}
         keys = models.key.Key.objects \
@@ -706,7 +714,7 @@ def users(self, request, *args, **kwargs):
                      .values('public', 'fingerprint') \
                      .order_by('created')
         if not keys:
-            raise NotFound("No Keys match the given query.")
+            return Response("No Keys match the given query.", status=status.HTTP_404_NOT_FOUND)
 
         for info in keys:
             data[request.user.username].append({
@@ -726,9 +734,9 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         self.user = request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(self.user, app, request.method)
+        has_permission, message = permissions.has_object_permission(self.user, app, request.method)
         if not has_permission:
-            raise PermissionDenied(message)
+            return Response(message, status=status.HTTP_403_FORBIDDEN)
         request.data['app'] = app
         request.data['owner'] = self.user
         super(BuildHookViewSet, self).create(request, *args, **kwargs)
@@ -753,56 +761,79 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(
+        has_permission, message = permissions.has_object_permission(
             request.user, app, request.method)
         if not has_permission:
-            raise PermissionDenied(message)
+            return Response(message, status=status.HTTP_403_FORBIDDEN)
         config = app.release_set.filter(failed=False).latest().config
         serializer = self.get_serializer(config)
         return Response(serializer.data, status=status.HTTP_200_OK)
 
 
-class AppPermsViewSet(BaseDryccViewSet):
+class UserPermViewSet(BaseDryccViewSet):
     """RESTful views for sharing apps with collaborators."""
 
-    model = models.app.App  # models class
-    perm = 'use_app'    # short name for permission
-
-    def get_queryset(self):
-        return self.model.objects.all()
+    def codes(self, request, **kwargs):
+        results = []
+        for _, object_policy in permissions.object_policy_registry.all():
+            results.append({
+                "codename": object_policy.codename,
+                "description": object_policy.description,
+            })
+        # fake out pagination for now
+        pagination = {'results': results, 'count': len(results)}
+        return Response(data=pagination)
 
     def list(self, request, **kwargs):
-        app = self.get_object()
-        perm_name = "api.{}".format(self.perm)
-        usernames = [u.username for u in get_users_with_perms(app)
-                     if u.has_perm(perm_name, app)]
-        return Response({'users': usernames})
+        cls, object_policy = permissions.object_policy_registry.get(
+            request.query_params.get('codename', ''))
+        if object_policy:
+            object_policies = [(cls, object_policy), ]
+        else:
+            object_policies = permissions.object_policy_registry.all()
+        results = []
+        for cls, object_policy in object_policies:
+            user_object_perms = UserObjectPermission.objects.filter(
+                object_pk__in=[obj.pk for obj in cls.objects.filter(owner=request.user)],
+                permission__codename=object_policy.codename
+            )
+            user_object_perms |= UserObjectPermission.objects.filter(
+                user=request.user, permission__codename=object_policy.codename)
+
+            for user_object_perm in user_object_perms:
+                results.append({
+                    "id": user_object_perm.pk,
+                    "codename": user_object_perm.permission.codename,
+                    "uniqueid": getattr(user_object_perm.content_object, object_policy.unique),
+                    "username": user_object_perm.user.username,
+                })
+        # fake out pagination for now
+        pagination = {'results': results, 'count': len(results)}
+        return Response(data=pagination)
 
     def create(self, request, **kwargs):
-        app = self.get_object()
-        if not permissions.IsOwnerOrAdmin.has_object_permission(permissions.IsOwnerOrAdmin(),
-                                                                request, self, app):
-            raise PermissionDenied()
-
-        user = get_object_or_404(User, username=request.data['username'])
-        assign_perm(self.perm, user, app)
-        app.log("User {} was granted access to {}".format(user, app))
+        username = request.data.get('username')
+        codename = request.data.get('codename')
+        uniqueid = request.data.get('uniqueid')
+        cls, object_policy = permissions.object_policy_registry.get(codename)
+        if not cls or not object_policy:
+            return Response(f"User {codename} not found", status=status.HTTP_404_NOT_FOUND)
+        obj = get_object_or_404(cls, **{object_policy.unique: uniqueid})
+        if not permissions.IsOwnerOrAdmin().has_object_permission(request, self, obj):
+            return Response(status=status.HTTP_403_FORBIDDEN)
+        user = get_object_or_404(User, username=username)
+        assign_perm(object_policy.codename, user, obj)
+        getattr(obj, "log", logger.info)("User {} was granted access to {}".format(user, obj))
         return Response(status=status.HTTP_201_CREATED)
 
     def destroy(self, request, **kwargs):
-        app = get_object_or_404(models.app.App, id=self.kwargs['id'])
-        user = get_object_or_404(User, username=kwargs['username'])
-
-        perm_name = "api.{}".format(self.perm)
-        if not user.has_perm(perm_name, app):
-            raise PermissionDenied()
-
-        if (user != request.user and
-            not permissions.IsOwnerOrAdmin.has_object_permission(permissions.IsOwnerOrAdmin(),
-                                                                 request, self, app)):
-            raise PermissionDenied()
-        remove_perm(self.perm, user, app)
-        app.log("User {} was revoked access to {}".format(user, app))
+        user_object_permission = get_object_or_404(UserObjectPermission, id=self.kwargs['id'])
+        obj = user_object_permission.content_object
+        if not (obj.owner == request.user or user_object_permission.user == request.user):
+            return Response(status=status.HTTP_403_FORBIDDEN)
+        perm_name = f"api.{user_object_permission.permission.codename}"
+        remove_perm(perm_name, user_object_permission.user, obj)
+        obj.log("User {} was revoked access to {}".format(user_object_permission.user, obj))
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
@@ -975,7 +1006,7 @@ def binding(self, request, *args, **kwargs):
             logger.info("resoruce unbind response data: {}".format(serializer))
             return Response(serializer.data)
         else:
-            return Http404("unknown action")
+            return Response("unknown action", status=status.HTTP_404_NOT_FOUND)
 
 
 class GatewayViewSet(AppResourceViewSet):
@@ -1098,30 +1129,6 @@ def delete(self, request, *args, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class AdminPermsViewSet(BaseDryccViewSet):
-    """RESTful views for sharing admin permissions with other users."""
-
-    model = User
-    serializer_class = serializers.AdminUserSerializer
-    permission_classes = [permissions.IsAdmin]
-
-    def get_queryset(self, **kwargs):
-        self.check_object_permissions(self.request, self.request.user)
-        return self.model.objects.filter(is_active=True, is_superuser=True)
-
-    def create(self, request, **kwargs):
-        user = get_object_or_404(self.model, username=request.data['username'])
-        user.is_superuser = user.is_staff = True
-        user.save(update_fields=['is_superuser', 'is_staff'])
-        return Response(status=status.HTTP_201_CREATED)
-
-    def destroy(self, request, **kwargs):
-        user = get_object_or_404(self.model, username=kwargs['username'])
-        user.is_superuser = user.is_staff = False
-        user.save(update_fields=['is_superuser', 'is_staff'])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
 class UserView(BaseDryccViewSet):
     """A Viewset for interacting with User objects."""
     model = User

From 697fb0d491418e53e633d478ed7191c2bc46ee6d Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 14 Aug 2024 18:05:12 +0800
Subject: [PATCH 056/200] feat(controller): support autorollback config (#154)

---
 .../0010_appsettings_autorollback.py           | 18 ++++++++++++++++++
 rootfs/api/models/app.py                       |  4 ++++
 rootfs/api/models/appsettings.py               | 12 +++++++++++-
 rootfs/api/models/release.py                   | 11 ++++++++++-
 rootfs/api/tests/test_app_settings.py          | 16 ++++++++++++++++
 5 files changed, 59 insertions(+), 2 deletions(-)
 create mode 100644 rootfs/api/migrations/0010_appsettings_autorollback.py

diff --git a/rootfs/api/migrations/0010_appsettings_autorollback.py b/rootfs/api/migrations/0010_appsettings_autorollback.py
new file mode 100644
index 000000000..aa4ab8f88
--- /dev/null
+++ b/rootfs/api/migrations/0010_appsettings_autorollback.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-08-13 07:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0009_remove_appsettings_canaries_remove_release_canary_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appsettings',
+            name='autorollback',
+            field=models.BooleanField(default=True),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 515ce2d33..a3d18fe37 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -240,6 +240,10 @@ def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
                         raise DryccException(f'pipeline run state error: {state}')
             if procfile_types is None or len(procfile_types) > 0:
                 self.log(f"{prefix} starts running pipeline.deploy")
+                app_settings = self.appsettings_set.latest()
+                if app_settings.autorollback is False:
+                    self.log(f"{prefix} deploy do not rollback on failure")
+                    rollback_on_failure = False
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
             else:
                 self.log(f"{prefix} no changes, skip executing pipeline.deploy")
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index 94f658257..c68346beb 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -20,6 +20,7 @@ class AppSettings(UuidAuditedModel):
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     routable = models.BooleanField(default=True)
+    autorollback = models.BooleanField(default=True)
     autoscale = models.JSONField(default=dict, blank=True)
     label = models.JSONField(default=dict, blank=True)
 
@@ -71,6 +72,15 @@ def _update_routable(self, previous_settings):
         elif old != new:
             self.summary += ["{} changed routablity from {} to {}".format(self.owner, old, new)]
 
+    def _update_autorollback(self, previous_settings):
+        old = getattr(previous_settings, 'autorollback', None)
+        new = getattr(self, 'autorollback', None)
+        # if nothing changed copy the settings from previous
+        if new is None and old is not None:
+            setattr(self, 'autorollback', old)
+        elif old != new:
+            self.summary += ["{} changed autorollback from {} to {}".format(self.owner, old, new)]
+
     def _update_autoscale(self, previous_settings):
         data = getattr(previous_settings, 'autoscale', {}).copy()
         new = getattr(self, 'autoscale', {}).copy()
@@ -149,7 +159,7 @@ def _update_fields(self, ignore_update_fields=None):
             previous_settings = self.app.appsettings_set.latest()
         except AppSettings.DoesNotExist:
             pass
-        update_fields = ["routable", "autoscale", "label"]
+        update_fields = ["routable", "autorollback", "autoscale", "label"]
         try:
             for update_field in update_fields:
                 if ignore_update_fields is None or update_field not in ignore_update_fields:
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 6c89222bc..baadb7ad0 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -2,6 +2,7 @@
 
 from django.conf import settings
 from django.db import models
+from django.db.models import Q
 from django.contrib.auth import get_user_model
 from api.utils import dict_diff
 from api.tasks import run_pipeline
@@ -9,6 +10,7 @@
 from scheduler import KubeHTTPException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 from .base import UuidAuditedModel
+from .appsettings import AppSettings
 
 
 User = get_user_model()
@@ -197,9 +199,16 @@ def previous(self):
         if self.pk:
             releases = releases.exclude(pk=self.pk)
 
+        q = Q(failed=False, state="succeed")
+        try:
+            app_settings = self.app.appsettings_set.latest()
+            if app_settings.autorollback is False:
+                q = Q(state__in=["crashed", "succeed"])
+        except AppSettings.DoesNotExist:
+            pass
         try:
             # Get the Release previous to this one
-            prev_release = releases.filter(failed=False, state="succeed").latest()
+            prev_release = releases.filter(q).latest()
         except Release.DoesNotExist:
             prev_release = None
         return prev_release
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
index 75cf0f2d3..d6db77e50 100644
--- a/rootfs/api/tests/test_app_settings.py
+++ b/rootfs/api/tests/test_app_settings.py
@@ -40,6 +40,22 @@ def test_settings_routable(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         self.assertFalse(app.appsettings_set.latest().routable)
 
+    def test_settings_autorollback(self, mock_requests):
+        """
+        Create an application with the autorollback flag turned on or off
+        """
+        # create app, expecting autorollback to be true
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+        # Set autorollback to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autorollback': False}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().autorollback)
+
     def test_autoscale(self, mock_requests):
         """
         Test that autoscale can be applied

From 51b85980138d3dca6b35f976ca7147fcac00220d Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 15 Aug 2024 09:59:05 +0800
Subject: [PATCH 057/200] chore(controller): merge migrations (#155)

---
 rootfs/api/migrations/0011_merge_20240815_0955.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 rootfs/api/migrations/0011_merge_20240815_0955.py

diff --git a/rootfs/api/migrations/0011_merge_20240815_0955.py b/rootfs/api/migrations/0011_merge_20240815_0955.py
new file mode 100644
index 000000000..adeb0d7d6
--- /dev/null
+++ b/rootfs/api/migrations/0011_merge_20240815_0955.py
@@ -0,0 +1,14 @@
+# Generated by Django 4.2.15 on 2024-08-15 01:55
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0010_alter_certificate_options'),
+        ('api', '0010_appsettings_autorollback'),
+    ]
+
+    operations = [
+    ]

From 0923495c96b35a34f41019e8bee9f670e4143411 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 16 Aug 2024 09:38:12 +0800
Subject: [PATCH 058/200] fix(controller): volume can not delete when ptype
 isn't exist

---
 rootfs/api/models/release.py | 8 ++++++--
 rootfs/api/models/volume.py  | 2 --
 rootfs/api/views.py          | 4 +++-
 3 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index baadb7ad0..f3a76c0c2 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -387,8 +387,12 @@ def _delete_release_in_scheduler(self, namespace, procfile_types, version_name):
         for replica_set in replica_sets:
             # Deployment takes care of this in the API, RS does not
             # Have the RS scale down pods and delete itself
-            self.scheduler().rs.scale(namespace, replica_set['metadata']['name'], 0, timeout)
-            self.scheduler().rs.delete(namespace, replica_set['metadata']['name'])
+            try:
+                self.scheduler().rs.scale(namespace, replica_set['metadata']['name'], 0, timeout)
+                self.scheduler().rs.delete(namespace, replica_set['metadata']['name'])
+            except KubeHTTPException as e:
+                if e.response.status_code != 404:
+                    raise
 
     def save(self, *args, **kwargs):  # noqa
         if not self.summary:
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 5cfe3ae5f..666e386d2 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -36,8 +36,6 @@ def save(self, *args, **kwargs):
 
     @transaction.atomic
     def delete(self, *args, **kwargs):
-        if self.path:
-            raise DryccException("the volume is not unmounted")
         # Deatch volume, updates k8s
         if self.type == "csi":
             self._delete_pvc()
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 5bebce5ce..bbd7291cf 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -855,7 +855,9 @@ def expand(self, request, **kwargs):
 
     def destroy(self, request, **kwargs):
         volume = self.get_object()
-        if volume.path != {}:
+        app = self.get_app()
+        is_subset = set(volume.path.keys()).issubset(set(app.procfile_types))
+        if volume.path != {} and is_subset:
             raise DryccException("this volume is mounting")
         volume.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)

From 20e8d2beb7ec4b99f21965717c73dfa83a7ad5fa Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 19 Aug 2024 18:47:19 +0800
Subject: [PATCH 059/200] chore(gateway): remove unnecessary else

---
 rootfs/api/models/gateway.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 704f515fa..c5cc5fa5d 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -119,8 +119,7 @@ def change_default_tls(self):
                            models.Q(certificate__isnull=False)).exists()
         if tls_enabled:
             return self.add(DEFAULT_HTTPS_PORT, "HTTPS")[0]
-        else:
-            return self.remove(DEFAULT_HTTPS_PORT, "HTTPS")[0]
+        return self.remove(DEFAULT_HTTPS_PORT, "HTTPS")[0]
 
     def save(self, *args, **kwargs):
         self.change_default_tls()

From 2c262927af51eca0576d60837bf21f3c01200b27 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 20 Aug 2024 17:37:36 +0800
Subject: [PATCH 060/200] chore(perms): destroy perms log err

---
 rootfs/api/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index bbd7291cf..7104676f8 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -833,7 +833,7 @@ def destroy(self, request, **kwargs):
             return Response(status=status.HTTP_403_FORBIDDEN)
         perm_name = f"api.{user_object_permission.permission.codename}"
         remove_perm(perm_name, user_object_permission.user, obj)
-        obj.log("User {} was revoked access to {}".format(user_object_permission.user, obj))
+        getattr(obj, "log", logger.info)("User {} was revoked access to {}".format(user_object_permission.user, obj))  # noqa
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 

From cae586a0850e53c379128910969d888d175de545 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sun, 25 Aug 2024 11:44:11 +0800
Subject: [PATCH 061/200] feat(release): add deploy release

---
 rootfs/api/models/app.py         | 44 ++++++++++++++++--------------
 rootfs/api/models/build.py       |  2 +-
 rootfs/api/models/release.py     |  7 ++---
 rootfs/api/tasks.py              | 11 ++------
 rootfs/api/tests/test_release.py | 31 ++++++++++++++++++++-
 rootfs/api/urls.py               |  3 +++
 rootfs/api/views.py              | 46 ++++++++++++++++++++++++--------
 7 files changed, 99 insertions(+), 45 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index a3d18fe37..e65b0c6bf 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -220,30 +220,35 @@ def scale(self, user, structure):
         app_settings = self.appsettings_set.latest()
         self._scale(user, structure, release, app_settings)
 
-    def pipeline(self, release, force_deploy=False, rollback_on_failure=True):
+    def prepare(self, release):
+        prefix = f"[pipeline] prepare {release.version_name}"
+        if release.build.dryccfile:
+            if 'run' in release.build.dryccfile and release.get_run_trigger():
+                self.log(f"{prefix} starts running pipeline.run")
+                job_name = self.run(
+                    self.owner, release.get_run_image(), command=release.get_run_command(),
+                    args=release.get_run_args(), timeout=release.get_run_timeout(),
+                    expires=release.get_run_timeout())
+                state, labels = 'initializing', {'job-name': job_name}
+                for count, state in enumerate(self.scheduler().pod.watch(
+                        self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
+                    self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
+                if state != 'down':
+                    raise DryccException(f'pipeline run state error: {state}')
+                return True
+        return False
+
+    def pipeline(self, release, force_deploy=False):
         prefix = f"[pipeline] release {release.version_name}"
         try:
             self.log(f"{prefix} starts running...")
             procfile_types = release.diff_procfile_types()
-            if release.build.dryccfile:
-                if 'run' in release.build.dryccfile and release.get_run_trigger():
-                    self.log(f"{prefix} starts running pipeline.run")
-                    job_name = self.run(
-                        self.owner, release.get_run_image(), command=release.get_run_command(),
-                        args=release.get_run_args(), timeout=release.get_run_timeout(),
-                        expires=release.get_run_timeout())
-                    state, labels = 'initializing', {'job-name': job_name}
-                    for count, state in enumerate(self.scheduler().pod.watch(
-                            self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
-                        self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
-                    if state != 'down':
-                        raise DryccException(f'pipeline run state error: {state}')
+            self.prepare(release)
             if procfile_types is None or len(procfile_types) > 0:
                 self.log(f"{prefix} starts running pipeline.deploy")
-                app_settings = self.appsettings_set.latest()
-                if app_settings.autorollback is False:
+                rollback_on_failure = self.appsettings_set.latest().autorollback
+                if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
-                    rollback_on_failure = False
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
             else:
                 self.log(f"{prefix} no changes, skip executing pipeline.deploy")
@@ -617,7 +622,7 @@ def state_to_k8s(self):
         if len(procfile_types) == 0:
             self.log('the cluster status is the latest, skipping deployment...')
             return
-        self.deploy(release, procfile_types)
+        self.deploy(release, procfile_types, False, False)
 
     def set_application_config(self, release, procfile_type):
         """
@@ -750,8 +755,7 @@ def _deploy(self, deploys, procfile_types, prev_release,
                     # This goes in the log before the rollback starts
                     self.log(err, logging.ERROR)
                     # revert all process types to old release
-                    self.deploy(prev_release, procfile_types,
-                                force_deploy=True, rollback_on_failure=False)
+                    self.deploy(prev_release, procfile_types, True, False)
                     # let it bubble up
                     raise DryccException('{}\n{}'.format(err, str(e))) from e
 
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index e8bb99d43..ff41784f1 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -79,7 +79,7 @@ def create_release(self, user, *args, **kwargs):
                 build=self,
                 config=latest_release.config,
             )
-            run_pipeline.delay(new_release)
+            run_pipeline.delay(new_release, force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index f3a76c0c2..5ffce7986 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -47,6 +47,10 @@ class Meta:
     def __str__(self):
         return "{0}-{1}".format(self.app.id, self.version_name)
 
+    @property
+    def deploying(self):
+        return self.build is not None and self.state == "created"
+
     @property
     def procfile_types(self):
         if self.build is not None:
@@ -222,11 +226,9 @@ def rollback(self, user, version=None):
                 raise DryccException('version cannot be below 0')
             elif version == 1:
                 raise DryccException('Cannot roll back to initial release.')
-
             prev = self.app.release_set.get(version=version)
             if prev.failed:
                 raise DryccException('Cannot roll back to failed release.')
-            app_settings = self.app.appsettings_set.latest()
             latest_version = self.app.release_set.latest().version
             new_release = self.new(
                 user,
@@ -234,7 +236,6 @@ def rollback(self, user, version=None):
                 config=prev.config,
                 summary="{} rolled back to v{}".format(user, version),
             )
-
             if self.build is not None:
                 run_pipeline.delay(new_release, force_deploy=True)
             return new_release
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 2056462cb..062e16b97 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -66,19 +66,12 @@ def run_pipeline(release, *args, **kwargs):
     autoretry_for=(ServiceUnavailable, ),
     retry_kwargs={'max_retries': None}
 )
-def run_deploy(release, config):
+def run_deploy(release, *args, **kwargs):
     task_id = uuid.uuid4().hex
     signals.request_started.send(sender=task_id)
     try:
         if release.build is not None:
-            procfile_types = set()
-            for field, diff in config.diff().items():
-                if field in config.procfile_fields:
-                    for value in diff.values():
-                        procfile_types.update(value.keys())
-            # all_diff_fields changed, deploy all.
-            procfile_types = procfile_types if procfile_types else None
-            config.app.deploy(release, procfile_types)
+            release.app.deploy(release, *args, **kwargs)
         release.state = "succeed"
         release.save()
     except Exception as e:
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index a16d98bd4..f8e69a2cf 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -199,6 +199,35 @@ def test_response_data(self, mock_requests):
         }
         self.assertEqual(response.data, response.data | expected)
 
+    def test_release_deploy(self, mock_requests):
+        app_id = self.create_app()
+        # try to rollback with only 1 release extant, expecting 400
+        url = f"/v2/apps/{app_id}/releases/deploy/"
+        # app.deploy
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.return_value = None
+            response = self.client.post(url, {"types": "web"})
+            self.assertEqual(response.status_code, 400)
+
+        # post a new build
+        build_url = f"/v2/apps/{app_id}/builds"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(build_url, body)
+        self.assertEqual(response.status_code, 201)
+        # app.deploy
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.return_value = None
+            response = self.client.post(url, {"types": "web"})
+            self.assertEqual(response.status_code, 204)
+
     def test_release_rollback(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
@@ -887,7 +916,7 @@ def test_diff_procfile_types(self, mock_requests):
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             with mock.patch('api.models.app.logger') as mock_logger:
                 mock_kube.return_value = ['up', 'down']
-                app.pipeline(release, False, True)
+                app.pipeline(release, False)
                 self.assertEqual(release.state, "succeed")
                 prefix = f"[{release.app.id}]: [pipeline] release v{release.version}"
                 exp_msg = f"{prefix} no changes, skip executing pipeline.deploy"
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 312aecf74..e894a3116 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -39,6 +39,9 @@
     re_path(
         r"^apps/(?P<id>{})/releases/v(?P<version>[0-9]+)/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'get': 'retrieve'})),
+    re_path(
+        r"^apps/(?P<id>{})/releases/deploy/?$".format(settings.APP_URL_REGEX),
+        views.ReleaseViewSet.as_view({'post': 'deploy'})),
     re_path(
         r"^apps/(?P<id>{})/releases/rollback/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'post': 'rollback'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 7104676f8..5f0d0a2df 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -387,13 +387,21 @@ class ConfigViewSet(ReleasableViewSet):
 
     def post_save(self, config):
         latest_release = config.app.release_set.filter(failed=False).latest()
-        if latest_release.build is not None and latest_release.state == "created":
+        if latest_release.deploying:
             config.delete()
             raise DryccException('There is an executing pipeline, please wait')
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
-            run_deploy.delay(release, config)
+            procfile_types = set()
+            for field, diff in config.diff().items():
+                if field in config.procfile_fields:
+                    for value in diff.values():
+                        procfile_types.update(value.keys())
+            # all_diff_fields changed, deploy all.
+            procfile_types = procfile_types if procfile_types else None
+            rollback_on_failure = config.app.appsettings_set.latest().autorollback
+            run_deploy.delay(release, procfile_types, False, rollback_on_failure)
         except Exception as e:
             config.delete()
             if isinstance(e, AlreadyExists):
@@ -451,15 +459,11 @@ def describe(self, *args, **kwargs):
 
     def restart(self, request, *args, **kwargs):
         app = self.get_app()
-        ptypes = []
-        types = request.data.get("types", "").split(",")
-        types = [ptype for ptype in set(types) if ptype != ""]
-        if not types:
-            # all ptypes need to restart
-            ptypes = app.structure.keys()
+        ptypes = set([ptype for ptype in request.data.get("types", "").split(",") if ptype])
+        if not ptypes:
+            ptypes = app.structure.keys()  # all ptypes need to restart
         else:
-            ptypes = [ptype for ptype in types if ptype in app.structure]
-            invalid_ptypes = set(types) - set(ptypes)
+            invalid_ptypes = ptypes.difference(app.structure)
             if len(invalid_ptypes) != 0:
                 raise DryccException("process type {} is not included in procfile".
                                      format(','.join(invalid_ptypes)))
@@ -470,7 +474,7 @@ def restart(self, request, *args, **kwargs):
     def scale(self, request, **kwargs):
         app = self.get_app()
         latest_release = app.release_set.filter(failed=False).latest()
-        if latest_release.build is not None and latest_release.state == "created":
+        if latest_release.deploying:
             raise DryccException('There is an executing pipeline, please wait')
         scale_app.delay(app, request.user, request.data)
         return Response(status=status.HTTP_204_NO_CONTENT)
@@ -629,12 +633,32 @@ def get_object(self, **kwargs):
         qs = self.get_queryset(**kwargs)
         return get_object_or_404(qs, version=self.kwargs['version'])
 
+    def deploy(self, request, **kwargs):
+        """Deploy the latest release"""
+        release = self.get_app().release_set.latest()
+        if release.deploying:
+            raise DryccException('There is an executing pipeline, please wait')
+        procfile_types = set(
+            [ptype for ptype in request.data.get("types", "").split(",") if ptype])
+        if not procfile_types:
+            procfile_types = release.app.structure.keys()  # all procfile_types need to deploy
+        else:
+            invalid_procfile_types = procfile_types.difference(release.app.structure)
+            if len(invalid_procfile_types) != 0:
+                raise DryccException("process type {} is not included in procfile".
+                                     format(','.join(invalid_procfile_types)))
+        rollback_on_failure = release.app.appsettings_set.latest().autorollback
+        run_deploy.delay(release, procfile_types, False, rollback_on_failure)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
     def rollback(self, request, **kwargs):
         """
         Create a new release as a copy of the state of the compiled slug and config vars of a
         previous release.
         """
         latest_release = self.get_app().release_set.filter(failed=False).latest()
+        if latest_release.deploying:
+            raise DryccException('There is an executing pipeline, please wait')
         new_release = latest_release.rollback(request.user, request.data.get('version', None))
         response = {'version': new_release.version}
         return Response(response, status=status.HTTP_201_CREATED)

From fead61e29ff23c76909a41695343725de3993f13 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 26 Aug 2024 13:11:25 +0800
Subject: [PATCH 062/200] chore(controller): change process type names to
 ptypes

---
 rootfs/api/tests/test_ptypes.py  | 4 ++--
 rootfs/api/tests/test_release.py | 4 ++--
 rootfs/api/views.py              | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
index c32cbb6fd..1c83ec1a7 100644
--- a/rootfs/api/tests/test_ptypes.py
+++ b/rootfs/api/tests/test_ptypes.py
@@ -101,12 +101,12 @@ def test_restart_ptypes(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # restart web and workers ptype deployments
-        body = {"types": "web,worker"}
+        body = {"ptypes": "web,worker"}
         response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id), body)
         self.assertEqual(response.status_code, 204, response.data)
 
         # restart invalid ptypes
-        body = {"types": "web1"}
+        body = {"ptypes": "web1"}
         response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id), body)
         self.assertEqual(response.status_code, 400, response.data)
 
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index f8e69a2cf..15a33a8fa 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -206,7 +206,7 @@ def test_release_deploy(self, mock_requests):
         # app.deploy
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.return_value = None
-            response = self.client.post(url, {"types": "web"})
+            response = self.client.post(url, {"ptypes": "web"})
             self.assertEqual(response.status_code, 400)
 
         # post a new build
@@ -225,7 +225,7 @@ def test_release_deploy(self, mock_requests):
         # app.deploy
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.return_value = None
-            response = self.client.post(url, {"types": "web"})
+            response = self.client.post(url, {"ptypes": "web"})
             self.assertEqual(response.status_code, 204)
 
     def test_release_rollback(self, mock_requests):
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 5f0d0a2df..a4afc4b24 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -459,7 +459,7 @@ def describe(self, *args, **kwargs):
 
     def restart(self, request, *args, **kwargs):
         app = self.get_app()
-        ptypes = set([ptype for ptype in request.data.get("types", "").split(",") if ptype])
+        ptypes = set([ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
         if not ptypes:
             ptypes = app.structure.keys()  # all ptypes need to restart
         else:
@@ -639,7 +639,7 @@ def deploy(self, request, **kwargs):
         if release.deploying:
             raise DryccException('There is an executing pipeline, please wait')
         procfile_types = set(
-            [ptype for ptype in request.data.get("types", "").split(",") if ptype])
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
         if not procfile_types:
             procfile_types = release.app.structure.keys()  # all procfile_types need to deploy
         else:

From f3ba668f2102948f887d6ddcec5755d2129bf579 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 27 Aug 2024 11:11:47 +0800
Subject: [PATCH 063/200] chore(release): add autodeploy flag

---
 .../migrations/0012_appsettings_autodeploy.py |  27 ++
 rootfs/api/models/app.py                      |  64 ++--
 rootfs/api/models/appsettings.py              |  28 +-
 rootfs/api/models/build.py                    |   3 +-
 rootfs/api/models/release.py                  |  82 +---
 rootfs/api/serializers/schemas/dryccfile.py   |  23 +-
 rootfs/api/tasks.py                           |  25 --
 rootfs/api/tests/test_app.py                  |   4 +-
 rootfs/api/tests/test_app_settings.py         |  16 +
 rootfs/api/tests/test_build.py                |  24 +-
 rootfs/api/tests/test_release.py              | 351 ++----------------
 rootfs/api/views.py                           |  48 ++-
 12 files changed, 191 insertions(+), 504 deletions(-)
 create mode 100644 rootfs/api/migrations/0012_appsettings_autodeploy.py

diff --git a/rootfs/api/migrations/0012_appsettings_autodeploy.py b/rootfs/api/migrations/0012_appsettings_autodeploy.py
new file mode 100644
index 000000000..5b5eb8d0d
--- /dev/null
+++ b/rootfs/api/migrations/0012_appsettings_autodeploy.py
@@ -0,0 +1,27 @@
+# Generated by Django 4.2.15 on 2024-08-27 02:18
+
+from django.db import migrations, models
+
+
+def migration_dryccfile(apps, schema_editor):
+    build_model = apps.get_model('api', 'Build')
+    for build in build_model.objects.all():
+        if build.dryccfile and isinstance(build.dryccfile.get('run'), dict):
+            build.dryccfile['run'] = [build.dryccfile['run']]
+            build.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_merge_20240815_0955'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appsettings',
+            name='autodeploy',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.RunPython(migration_dryccfile),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index e65b0c6bf..311206b1e 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -113,6 +113,19 @@ def save(self, *args, **kwargs):
     def procfile_types(self):
         return list(self.structure.keys())
 
+    def check_procfile_types(self, procfile_types):
+        """
+        check available procfile types
+        """
+        if not procfile_types:
+            procfile_types = self.procfile_types
+        else:
+            invalid_procfile_types = procfile_types.difference(self.procfile_types)
+            if len(invalid_procfile_types) != 0:
+                raise DryccException("process type {} is not included in procfile".
+                                     format(','.join(invalid_procfile_types)))
+        return procfile_types
+
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -220,38 +233,28 @@ def scale(self, user, structure):
         app_settings = self.appsettings_set.latest()
         self._scale(user, structure, release, app_settings)
 
-    def prepare(self, release):
-        prefix = f"[pipeline] prepare {release.version_name}"
-        if release.build.dryccfile:
-            if 'run' in release.build.dryccfile and release.get_run_trigger():
-                self.log(f"{prefix} starts running pipeline.run")
-                job_name = self.run(
-                    self.owner, release.get_run_image(), command=release.get_run_command(),
-                    args=release.get_run_args(), timeout=release.get_run_timeout(),
-                    expires=release.get_run_timeout())
-                state, labels = 'initializing', {'job-name': job_name}
-                for count, state in enumerate(self.scheduler().pod.watch(
-                        self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
-                    self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
-                if state != 'down':
-                    raise DryccException(f'pipeline run state error: {state}')
-                return True
-        return False
-
-    def pipeline(self, release, force_deploy=False):
+    def pipeline(self, release, procfile_types=None, force_deploy=False):
         prefix = f"[pipeline] release {release.version_name}"
         try:
-            self.log(f"{prefix} starts running...")
-            procfile_types = release.diff_procfile_types()
-            self.prepare(release)
-            if procfile_types is None or len(procfile_types) > 0:
-                self.log(f"{prefix} starts running pipeline.deploy")
+            if release.build is not None:
+                if release.build.dryccfile:
+                    for run in release.get_runners(procfile_types):
+                        self.log(f"{prefix} starts running pipeline.run {run['image']}")
+                        job_name = self.run(
+                            self.owner, run['image'], command=run['command'],
+                            args=run['args'], timeout=run['timeout'], expires=run['timeout']
+                        )
+                        state, labels = 'initializing', {'job-name': job_name}
+                        for count, state in enumerate(self.scheduler().pod.watch(
+                                self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
+                            self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
+                        if state != 'down':
+                            raise DryccException(f'pipeline run state error: {state}')
+                self.log(f"{prefix} starts running...")
                 rollback_on_failure = self.appsettings_set.latest().autorollback
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
-            else:
-                self.log(f"{prefix} no changes, skip executing pipeline.deploy")
             release.state = "succeed"
         except Exception as e:
             release.state = "crashed"
@@ -356,14 +359,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         try:
             # create application config and build the pod manifest
             self.set_application_config(release, PROCFILE_TYPE_RUN)
-            self.scheduler().job.create(
-                self.id,
-                name,
-                image if image else release.get_run_image(),
-                command if command else release.get_run_command(),
-                args if args else release.get_run_args(),
-                **kwargs
-            )
+            self.scheduler().job.create(self.id, name, image, command, args, **kwargs)
         except Exception as e:
             err = '{} ({}): {}'.format(name, PROCFILE_TYPE_RUN, e)
             raise ServiceUnavailable(err) from e
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index c68346beb..61282ad81 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -20,6 +20,7 @@ class AppSettings(UuidAuditedModel):
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     routable = models.BooleanField(default=True)
+    autodeploy = models.BooleanField(default=True)
     autorollback = models.BooleanField(default=True)
     autoscale = models.JSONField(default=dict, blank=True)
     label = models.JSONField(default=dict, blank=True)
@@ -63,23 +64,14 @@ def previous(self):
             prev_app_settings = None
         return prev_app_settings
 
-    def _update_routable(self, previous_settings):
-        old = getattr(previous_settings, 'routable', None)
-        new = getattr(self, 'routable', None)
+    def _update_field(self, field, previous_settings):
+        old = getattr(previous_settings, field, None)
+        new = getattr(self, field, None)
         # if nothing changed copy the settings from previous
         if new is None and old is not None:
-            setattr(self, 'routable', old)
+            setattr(self, field, old)
         elif old != new:
-            self.summary += ["{} changed routablity from {} to {}".format(self.owner, old, new)]
-
-    def _update_autorollback(self, previous_settings):
-        old = getattr(previous_settings, 'autorollback', None)
-        new = getattr(self, 'autorollback', None)
-        # if nothing changed copy the settings from previous
-        if new is None and old is not None:
-            setattr(self, 'autorollback', old)
-        elif old != new:
-            self.summary += ["{} changed autorollback from {} to {}".format(self.owner, old, new)]
+            self.summary += ["{} changed {} from {} to {}".format(self.owner, field, old, new)]
 
     def _update_autoscale(self, previous_settings):
         data = getattr(previous_settings, 'autoscale', {}).copy()
@@ -159,11 +151,15 @@ def _update_fields(self, ignore_update_fields=None):
             previous_settings = self.app.appsettings_set.latest()
         except AppSettings.DoesNotExist:
             pass
-        update_fields = ["routable", "autorollback", "autoscale", "label"]
+        update_fields = ["routable", "autodeploy", "autorollback", "autoscale", "label"]
         try:
             for update_field in update_fields:
                 if ignore_update_fields is None or update_field not in ignore_update_fields:
-                    getattr(self, "_update_%s" % update_field)(previous_settings)
+                    method = getattr(self, "_update_%s" % update_field, None)
+                    if method:
+                        method(previous_settings)
+                    else:
+                        self._update_field(update_field, previous_settings)
         except (UnprocessableEntity, NotFound):
             raise
         except Exception as e:
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index ff41784f1..660a08a7b 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -79,7 +79,8 @@ def create_release(self, user, *args, **kwargs):
                 build=self,
                 config=latest_release.config,
             )
-            run_pipeline.delay(new_release, force_deploy=False)
+            if self.app.appsettings_set.latest().autorollback:
+                run_pipeline.delay(new_release, force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 5ffce7986..8ffc51247 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -4,7 +4,6 @@
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth import get_user_model
-from api.utils import dict_diff
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
@@ -61,43 +60,22 @@ def procfile_types(self):
     def version_name(self):
         return f'v{self.version}'
 
-    def get_run_image(self):
-        """
-        In the run phase of dryccfile
-        Return the kubernetes "container image" to be sent off to the scheduler.
-        """
-        image = self.build.dryccfile.get('run', {}).get(
-            'image', self.build.get_image('run'))
-        return self.build.get_image(image, default_image=image)
-
-    def get_run_args(self):
-        """
-        In the run phase of dryccfile
-        Return the kubernetes "container arguments" to be sent off to the scheduler.
-        """
-        return self.build.dryccfile.get('run', {}).get('args', [])
-
-    def get_run_command(self):
-        """
-        In the run phase of dryccfile
-        Return the kubernetes "container command" to be sent off to the scheduler.
-        """
-        return self.build.dryccfile.get('run', {}).get('command', [])
-
-    def get_run_timeout(self):
-        return int(self.build.dryccfile.get('run', {}).get(
-            'timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT))
-
-    def get_run_trigger(self):
-        if 'ptypes' not in self.build.dryccfile.get('run', {}).get('when', {}):
-            return True
-        procfile_types = self.diff_procfile_types()
-        if procfile_types is None:
-            return True
-        for procfile_type in procfile_types:
-            if procfile_type in self.build.dryccfile['run']['when']['ptypes']:
-                return True
-        return False
+    def get_runners(self, procfile_types):
+        results = []
+        procfile_types = self.procfile_types if not procfile_types else procfile_types
+        for run in self.build.dryccfile.get('run', []):
+            for container_type in procfile_types:
+                when_ptypes = run.get('when', {}).get('ptypes', [])
+                if not when_ptypes or container_type in when_ptypes:
+                    image = run.get('image', self.build.get_image(container_type))
+                    results.append({
+                        'image': self.build.get_image(image, default_image=image),
+                        'args': run.get('args', []),
+                        'command': run.get('command', []),
+                        'timeout': run.get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
+                    })
+                    break
+        return results
 
     def get_deploy_image(self, container_type):
         """
@@ -133,30 +111,6 @@ def get_deploy_command(self, container_type):
         return self.build.dryccfile.get(
             'deploy', {}).get(container_type, {}).get('command', [])
 
-    def diff_procfile_types(self):
-        """
-        If returning None, it indicates that all procfile_types have changed.
-        """
-        def _get_full_deploy(release):
-            deploy = {}
-            for procfile_type, value in release.build.dryccfile['deploy'].items():
-                value['image'] = release.get_deploy_image(procfile_type)
-                value['args'] = release.get_deploy_args(procfile_type)
-                value['command'] = release.get_deploy_command(procfile_type)
-                deploy[procfile_type] = value
-            return deploy
-
-        pre_release = self.previous()
-        if (pre_release and pre_release.build and
-                pre_release.build.dryccfile and self.build and self.build.dryccfile):
-            deploy = _get_full_deploy(self)
-            pre_deploy = _get_full_deploy(pre_release)
-            procfile_types = set()
-            for value in dict_diff(deploy, pre_deploy).values():
-                procfile_types = procfile_types.union(value.keys())
-            return procfile_types
-        return None
-
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
@@ -217,7 +171,7 @@ def previous(self):
             prev_release = None
         return prev_release
 
-    def rollback(self, user, version=None):
+    def rollback(self, user, procfile_types=None, version=None):
         try:
             # if no version is provided then grab version from object
             version = (self.version - 1) if version is None else int(version)
@@ -237,7 +191,7 @@ def rollback(self, user, version=None):
                 summary="{} rolled back to v{}".format(user, version),
             )
             if self.build is not None:
-                run_pipeline.delay(new_release, force_deploy=True)
+                run_pipeline.delay(new_release, procfile_types, force_deploy=True)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
diff --git a/rootfs/api/serializers/schemas/dryccfile.py b/rootfs/api/serializers/schemas/dryccfile.py
index 3ad8efab8..c1554f4e3 100644
--- a/rootfs/api/serializers/schemas/dryccfile.py
+++ b/rootfs/api/serializers/schemas/dryccfile.py
@@ -18,17 +18,20 @@
             },
         },
         "run": {
-            "type": "object",
-            "properties": {
-                "image": {"type": "string"},
-                "command": {
-                    "type": "array",
-                    "items": {"type": "string"},
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "image": {"type": "string"},
+                    "command": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                    },
+                    "args": {
+                        "type": "array",
+                        "items": {"type": "string"},
+                    }
                 },
-                "args": {
-                    "type": "array",
-                    "items": {"type": "string"},
-                }
             },
         },
         "deploy": {
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 062e16b97..73bf0154c 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -62,31 +62,6 @@ def run_pipeline(release, *args, **kwargs):
         signals.request_finished.send(sender=task_id)
 
 
-@shared_task(
-    autoretry_for=(ServiceUnavailable, ),
-    retry_kwargs={'max_retries': None}
-)
-def run_deploy(release, *args, **kwargs):
-    task_id = uuid.uuid4().hex
-    signals.request_started.send(sender=task_id)
-    try:
-        if release.build is not None:
-            release.app.deploy(release, *args, **kwargs)
-        release.state = "succeed"
-        release.save()
-    except Exception as e:
-        release.state = "crashed"
-        release.failed = True
-        if release.summary:
-            release.summary += " "
-        release.summary += "{} deployed a config that failed".format(release.owner)
-        # Get the exception that has occured
-        release.exception = "error: {}".format(str(e))
-        release.save()
-    finally:
-        signals.request_finished.send(sender=task_id)
-
-
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_jitter=True,
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 25b81da22..4d880266f 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -235,8 +235,8 @@ def test_run_without_release_should_error(self, mock_requests):
         body = {'command': 'ls -al'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {'detail': 'No build associated with this '
-                                                   'release to run this command'})
+        self.assertEqual(
+            str(response.data["detail"]), 'no build available, please deploy a release')
 
     @mock.patch('api.models.app.App.run', _mock_run)
     @mock.patch('api.models.app.App.deploy', mock_none)
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
index d6db77e50..7f4b621bf 100644
--- a/rootfs/api/tests/test_app_settings.py
+++ b/rootfs/api/tests/test_app_settings.py
@@ -40,6 +40,22 @@ def test_settings_routable(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         self.assertFalse(app.appsettings_set.latest().routable)
 
+    def test_settings_autodeploy(self, mock_requests):
+        """
+        Create an application with the autorollback flag turned on or off
+        """
+        # create app, expecting autodeploy to be true
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        # Set autodeploy to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autodeploy': False}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
+
     def test_settings_autorollback(self, mock_requests):
         """
         Create an application with the autorollback flag turned on or off
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index e06cb9d1f..383eb9b2f 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -739,10 +739,12 @@ def test_dryccfile_ok(self, mock_requests):
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
                     "config": {"RAILS_ENV": "development", "FOO": "bar"}
                 },
-                "run": {
-                    "command": ["./deployment-tasks.sh"],
-                    "image": run_image,
-                },
+                "run": [
+                    {
+                        "command": ["./deployment-tasks.sh"],
+                        "image": run_image,
+                    }
+                ],
                 "deploy": {
                     "web": {
                         "command": ["bash", "-c"],
@@ -773,7 +775,9 @@ def test_dryccfile_ok(self, mock_requests):
             self.assertEqual(release["state"], "succeed", data)
             self.assertEqual(release["version"], 2, data)
             release_obj = app.release_set.filter(version=release["version"])[0]
-            self.assertEqual(release_obj.get_run_image(), run_image, data)
+            runners = release_obj.get_runners(["web"])
+            self.assertEqual(len(runners), 1)
+            self.assertEqual(runners[0]["image"], run_image, data)
             self.assertEqual(release_obj.get_deploy_image("web"), web_image, data)
             self.assertEqual(release_obj.get_deploy_image("worker"), worker_image, data)
             self.assertEqual(release_obj.get_deploy_image("noexist"), default_image, data)
@@ -811,10 +815,12 @@ def test_dryccfile_format(self, mock_requests):
             }
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
-            body['dryccfile']['run'] = {
-                'command': ["bash", "-c"],
-                'args': ["ls /"]
-            }
+            body['dryccfile']['run'] = [
+                {
+                    'command': ["bash", "-c"],
+                    'args': ["ls /"]
+                }
+            ]
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
             body['dryccfile']['deploy'] = {}
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 15a33a8fa..8aa790fdb 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -117,10 +117,12 @@ def test_get_image(self, mock_requests):
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
                     "config": {"RAILS_ENV": "development", "FOO": "bar"}
                 },
-                "run": {
-                    "command": ["./deployment-tasks.sh"],
-                    "image": "worker",
-                },
+                "run": [
+                    {
+                        "command": ["./deployment-tasks.sh"],
+                        "image": "worker",
+                    }
+                ],
                 "deploy": {
                     "web": {
                         "command": ["bash", "-c"],
@@ -157,8 +159,10 @@ def test_get_image(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
             app = App.objects.get(id=app_id)
             release_obj = app.release_set.filter(version=2)[0]
+            runners = release_obj.get_runners(["web"])
+            self.assertEqual(len(runners), 1, release_obj.build.dryccfile)
             self.assertEqual(
-                release_obj.get_run_image(),
+                runners[0]["image"],
                 worker_image, release_obj.build.dryccfile)
             self.assertEqual(
                 release_obj.get_deploy_image("web"),
@@ -628,310 +632,25 @@ def test_release_external_registry(self, mock_requests):
         release = app.release_set.latest()
         self.assertEqual(release.get_port('web'), 9000)
 
-    def test_diff_procfile_types(self, mock_requests):
+    @override_settings(DRYCC_PILELINE_RUN_TIMEOUT=600)
+    def test_run(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
         user = User.objects.get(username='autotest')
-
-        # CNCF Buildpack app
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "build": {
-                    "docker": {
-                        "web": "Dockerfile",
-                        "worker": "worker/Dockerfile",
-                    }
-                },
-                "deploy": {
-                    "web": {
-                        "command": ["bash", "-ec"],
-                        "args": ["bundle exec puma -C config/puma.rb"]
-                    },
-                    "worker": {
-                        "command": ["bash", "-ec"],
-                        "args": ["python myworker.py"]
-                    }
-                }
-            },
-        )
-
-        # create an initial release
-        release = Release.objects.create(
-            version=2,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertIsNone(release.diff_procfile_types())
-        # test has image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "deploy": {
-                    "web": {
-                        "image": "docker.io/test/test:v1",
-                        "command": ["bash", "-ec"],
-                        "args": ["bundle exec puma -C config/puma.rb"]
-                    },
-                    "worker": {
-                        "image": "docker.io/test/test:v1",
-                        "command": ["bash", "-ec"],
-                        "args": ["python myworker.py"]
-                    }
-                }
-            },
-        )
-        release = Release.objects.create(
-            version=3,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), {'web', 'worker'})
-        # test has image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "deploy": {
-                    "web": {
-                        "image": "docker.io/test/test:v2",
-                        "command": ["bash", "-ec"],
-                        "args": ["bundle exec puma -C config/puma.rb"]
-                    },
-                    "worker": {
-                        "image": "docker.io/test/test:v1",
-                        "command": ["bash", "-ec"],
-                        "args": ["python myworker.py"]
-                    }
-                }
-            },
-        )
-        release = Release.objects.create(
-            version=4,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), {'web'})
-        # test has image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "deploy": {
-                    "web": {
-                        "image": "docker.io/test/test:v2",
-                        "command": ["bash", "-ec"],
-                        "args": ["bundle exec puma -C config/puma.rb"]
-                    },
-                    "worker": {
-                        "image": "docker.io/test/test:v2",
-                        "command": ["bash", "-ec"],
-                        "args": ["python myworker.py"]
-                    }
-                }
-            },
-        )
-        release = Release.objects.create(
-            version=5,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), {'worker'})
-        # test no image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "build": {
-                    "docker": {
-                        "web": "Dockerfile",
-                        "worker": "worker/Dockerfile",
-                    }
-                },
-                "deploy": {
-                    "web": {
-                        "command": ["bash", "-ec"],
-                        "args": ["bundle exec puma -C config/puma.rb"]
-                    },
-                    "worker": {
-                        "command": ["bash", "-ec"],
-                        "args": ["python myworker.py"]
-                    },
-                    "worker-sync": {
-                        "image": "web",
-                        "command": ["bash", "-ec"],
-                        "args": ["python myworker.py sync"]
-                    },
-                }
-            },
-        )
-        release = Release.objects.create(
-            version=6,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), {'web', 'worker', 'worker-sync'})
-        # test has image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "run": {
-                    "args": ["sleep", "60s"],
-                    "image": "registry.drycc.cc/drycc/base:bookworm"
-                },
-                "deploy": {
-                    "web": {
-                        "image": "registry.drycc.cc/drycc/python-dev",
-                        "args": ["python", "-m", "http.server", "5000"]
-                    },
-                    "task": {
-                        "image": "docker.io/library/nginx",
-                        "command": ["sleep"],
-                        "args": ["infinity"]
-                    }
-                }
-            },
-        )
-        release = Release.objects.create(
-            version=7,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), {'worker', 'task', 'worker-sync', 'web'})
-        # test has image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "run": {
+        dryccfile = {
+            "run": [
+                {
                     "args": ["sleep", "60s"],
-                    "image": "registry.drycc.cc/drycc/base:bookworm"
+                    "image": "registry.drycc.cc/drycc/base:bookworm-web",
+                    "when": {"ptypes": ["web"]}
                 },
-                "deploy": {
-                    "web": {
-                        "image": "registry.drycc.cc/drycc/python-dev",
-                        "args": ["python", "-m", "http.server", "5000"]
-                    },
-                    "task": {
-                        "image": "docker.io/library/nginx:mainline-bookworm-perl",
-                        "command": ["sleep"],
-                        "args": ["infinity"]
-                    }
-                }
-            },
-        )
-        release = Release.objects.create(
-            version=8,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), {'task'})
-        # test has image
-        build = Build.objects.create(
-            owner=user,
-            app=app,
-            image="qwerty",
-            procfile={},
-            sha='african-swallow',
-            dockerfile={},
-            dryccfile={
-                "run": {
+                {
                     "args": ["sleep", "60s"],
-                    "image": "registry.drycc.cc/drycc/base:bookworm"
-                },
-                "deploy": {
-                    "web": {
-                        "image": "registry.drycc.cc/drycc/python-dev",
-                        "args": ["python", "-m", "http.server", "5000"]
-                    },
-                    "task": {
-                        "image": "docker.io/library/nginx:mainline-bookworm-perl",
-                        "command": ["sleep"],
-                        "args": ["infinity"]
-                    }
+                    "image": "registry.drycc.cc/drycc/base:bookworm-task",
+                    "when": {"ptypes": ["task"]},
+                    "timeout": 3000
                 }
-            },
-        )
-        release = Release.objects.create(
-            version=9,
-            owner=user,
-            app=app,
-            config=app.config_set.latest(),
-            build=build,
-            state="succeed"
-        )
-        self.assertEqual(release.diff_procfile_types(), set())
-        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
-            with mock.patch('api.models.app.logger') as mock_logger:
-                mock_kube.return_value = ['up', 'down']
-                app.pipeline(release, False)
-                self.assertEqual(release.state, "succeed")
-                prefix = f"[{release.app.id}]: [pipeline] release v{release.version}"
-                exp_msg = f"{prefix} no changes, skip executing pipeline.deploy"
-                mock_logger.log.assert_any_call(logging.INFO, exp_msg)
-
-    @override_settings(DRYCC_PILELINE_RUN_TIMEOUT=600)
-    def test_run(self, mock_requests):
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
-        dryccfile = {
-            "run": {
-                "args": ["sleep", "60s"],
-                "image": "registry.drycc.cc/drycc/base:bookworm"
-            },
+            ],
             "deploy": {
                 "web": {
                     "image": "registry.drycc.cc/drycc/python-dev",
@@ -950,11 +669,16 @@ def test_run(self, mock_requests):
         release = Release.objects.create(
             version=2, owner=user, app=app, config=app.config_set.latest(),
             build=build, state="succeed")
-        self.assertEqual(release.get_run_timeout(), 600)
-        self.assertEqual(release.get_run_trigger(), True)
+        runners = release.get_runners(["web"])
+        self.assertEqual(len(runners), 1)
+        self.assertEqual(runners[0]["timeout"], 600)
+
+        runners = release.get_runners(["task"])
+        self.assertEqual(len(runners), 1)
+        self.assertEqual(runners[0]["timeout"], 3000)
+
         # change timeout
-        dryccfile['run']['when'] = {'ptypes': ['web']}
-        dryccfile['run']['timeout'] = 3600
+        dryccfile['run'][0]['timeout'] = 3600
         dryccfile['deploy']['web']['image'] = 'registry.drycc.cc/drycc/python-dev:latest'
         build = Build.objects.create(
             owner=user, app=app, image="qwerty", procfile={},
@@ -962,15 +686,6 @@ def test_run(self, mock_requests):
         release = Release.objects.create(
             version=3, owner=user, app=app, config=app.config_set.latest(),
             build=build, state="succeed")
-        self.assertEqual(release.get_run_timeout(), 3600)
-        self.assertEqual(release.get_run_trigger(), True)
-        # change task image
-        dryccfile['deploy']['task']['image'] = 'docker.io/library/nginx:latest'
-        build = Build.objects.create(
-            owner=user, app=app, image="qwerty", procfile={},
-            sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
-        release = Release.objects.create(
-            version=4, owner=user, app=app, config=app.config_set.latest(),
-            build=build, state="succeed")
-        self.assertEqual(release.get_run_timeout(), 3600)
-        self.assertEqual(release.get_run_trigger(), False)
+        runners = release.get_runners(["web"])
+        self.assertEqual(len(runners), 1)
+        self.assertEqual(runners[0]["timeout"], 3600)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index a4afc4b24..e7c2bcf36 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -26,7 +26,7 @@
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
-    delete_pod, run_deploy
+    delete_pod, run_pipeline
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
@@ -306,6 +306,7 @@ def list(self, request, *args, **kwargs):
 
     def run(self, request, **kwargs):
         app = self.get_object()
+        ptype = request.data.get('ptype', 'run')
         command = request.data.get('command', '').split()
         timeout = int(request.data.get('timeout', 3600))
         expires = int(request.data.get('expires', 3600))
@@ -313,11 +314,15 @@ def run(self, request, **kwargs):
             expires = settings.KUBERNETES_JOB_MAX_TTL_SECONDS_AFTER_FINISHED
         if not command:
             raise DryccException('command is a required field, or it can be defined in Procfile')
+        release = app.release_set.filter(failed=False).latest()
+        if release.build is None:
+            raise DryccException('no build available, please deploy a release')
         volumes = request.data.get('volumes', None)
         if volumes:
             volumes = serializers.VolumeSerializer().validate_path(volumes)
-        app.run(self.request.user, args=command,
-                volumes=volumes, timeout=timeout, expires=expires)
+        app.run(self.request.user, release.get_deploy_image(ptype),
+                command=release.get_deploy_command(ptype), args=command, volumes=volumes,
+                timeout=timeout, expires=expires)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def update(self, request, **kwargs):
@@ -386,6 +391,8 @@ class ConfigViewSet(ReleasableViewSet):
     serializer_class = serializers.ConfigSerializer
 
     def post_save(self, config):
+        if not config.app.appsettings_set.latest().autorollback:
+            return
         latest_release = config.app.release_set.filter(failed=False).latest()
         if latest_release.deploying:
             config.delete()
@@ -400,8 +407,7 @@ def post_save(self, config):
                         procfile_types.update(value.keys())
             # all_diff_fields changed, deploy all.
             procfile_types = procfile_types if procfile_types else None
-            rollback_on_failure = config.app.appsettings_set.latest().autorollback
-            run_deploy.delay(release, procfile_types, False, rollback_on_failure)
+            run_pipeline.delay(release, procfile_types, False)
         except Exception as e:
             config.delete()
             if isinstance(e, AlreadyExists):
@@ -459,16 +465,11 @@ def describe(self, *args, **kwargs):
 
     def restart(self, request, *args, **kwargs):
         app = self.get_app()
-        ptypes = set([ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        if not ptypes:
-            ptypes = app.structure.keys()  # all ptypes need to restart
-        else:
-            invalid_ptypes = ptypes.difference(app.structure)
-            if len(invalid_ptypes) != 0:
-                raise DryccException("process type {} is not included in procfile".
-                                     format(','.join(invalid_ptypes)))
-        for ptype in set(ptypes):
-            restart_app.delay(app, **{"type": ptype})
+        procfile_types = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        procfile_types = app.check_procfile_types(procfile_types)
+        for procfile_type in set(procfile_types):
+            restart_app.delay(app, **{"type": procfile_type})
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def scale(self, request, **kwargs):
@@ -640,15 +641,8 @@ def deploy(self, request, **kwargs):
             raise DryccException('There is an executing pipeline, please wait')
         procfile_types = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        if not procfile_types:
-            procfile_types = release.app.structure.keys()  # all procfile_types need to deploy
-        else:
-            invalid_procfile_types = procfile_types.difference(release.app.structure)
-            if len(invalid_procfile_types) != 0:
-                raise DryccException("process type {} is not included in procfile".
-                                     format(','.join(invalid_procfile_types)))
-        rollback_on_failure = release.app.appsettings_set.latest().autorollback
-        run_deploy.delay(release, procfile_types, False, rollback_on_failure)
+        procfile_types = release.app.check_procfile_types(procfile_types)
+        run_pipeline.delay(release, procfile_types, False)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def rollback(self, request, **kwargs):
@@ -659,7 +653,11 @@ def rollback(self, request, **kwargs):
         latest_release = self.get_app().release_set.filter(failed=False).latest()
         if latest_release.deploying:
             raise DryccException('There is an executing pipeline, please wait')
-        new_release = latest_release.rollback(request.user, request.data.get('version', None))
+        procfile_types = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        procfile_types = latest_release.app.check_procfile_types(procfile_types)
+        new_release = latest_release.rollback(
+            request.user, procfile_types, request.data.get('version', None))
         response = {'version': new_release.version}
         return Response(response, status=status.HTTP_201_CREATED)
 

From d0d1dd801e2afc4ecae9a91f6128bbf11b7d5b81 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 28 Aug 2024 10:24:05 +0800
Subject: [PATCH 064/200] fix(controller): fix pod unhealthy do not rollback

---
 rootfs/scheduler/resources/pod.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 82a3a880a..b9695c38b 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -748,7 +748,7 @@ def _handle_not_ready_pods(self, namespace, labels):
             for event in self.events(pod):
                 if event['reason'] == 'Unhealthy':
                     # strip out whitespaces on either side
-                    message = "\n".join([x.strip() for x in event['message'].split("\n")])
+                    message = "\n".join([x.strip() for x in event['note'].split("\n")])
                     raise KubeException(message)
 
     def _get_probe_timeout(self, probe):

From 2e196bfcd2c850624900f52e85f45701330e3c8a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 28 Aug 2024 14:33:44 +0800
Subject: [PATCH 065/200] feat(release): add release deploy

---
 rootfs/api/models/app.py         | 19 +++++-----
 rootfs/api/models/build.py       |  8 ++---
 rootfs/api/models/release.py     | 35 +++++++++++++++----
 rootfs/api/tests/test_config.py  |  8 ++---
 rootfs/api/tests/test_gateway.py |  2 +-
 rootfs/api/tests/test_lock.py    | 48 +++++++++++++++++++++++++
 rootfs/api/tests/test_pods.py    | 12 ++++---
 rootfs/api/tests/test_release.py | 30 +++++++++++++++-
 rootfs/api/utils.py              | 60 ++++++++++++++++++++++++++++++--
 rootfs/api/views.py              | 27 +++++---------
 10 files changed, 198 insertions(+), 51 deletions(-)
 create mode 100644 rootfs/api/tests/test_lock.py

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 311206b1e..9fafcdf76 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -21,7 +21,7 @@
 
 from api.utils import get_session
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
-from api.utils import generate_app_name, apply_tasks
+from api.utils import DeployLock, generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
 from .gateway import Gateway, Route
 from .limit import LimitPlan
@@ -255,16 +255,17 @@ def pipeline(self, release, procfile_types=None, force_deploy=False):
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
                 self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
-            release.state = "succeed"
+            if release.state == "created":
+                release.state = "succeed"
+            release.add_condition(state="succeed", action="pipeline", ptypes=procfile_types)
         except Exception as e:
-            release.state = "crashed"
-            release.failed = True
-            if release.summary:
-                release.summary += " "
-            release.summary += "{} pipeline a release that failed".format(self.owner)
-            release.exception = "error: {}".format(str(e))
+            release.failed, release.state = True, "crashed"
+            release.add_condition(
+                state="crashed", action="pipeline", ptypes=procfile_types, exception=str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
-        release.save()
+        finally:
+            DeployLock(self.pk).release(procfile_types)
+            release.save(update_fields=["state", "failed"])  # avoid overwriting other fields
         self.log(f"{prefix} run completed...")
 
     def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_failure=True):
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 660a08a7b..331b3cf9c 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -3,7 +3,6 @@
 from django.db import models
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException, Conflict
-from api.tasks import run_pipeline
 from .base import UuidAuditedModel
 
 User = get_user_model()
@@ -79,8 +78,8 @@ def create_release(self, user, *args, **kwargs):
                 build=self,
                 config=latest_release.config,
             )
-            if self.app.appsettings_set.latest().autorollback:
-                run_pipeline.delay(new_release, force_deploy=False)
+            if self.app.appsettings_set.latest().autodeploy:
+                new_release.deploy(force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
@@ -95,7 +94,8 @@ def create_release(self, user, *args, **kwargs):
                     self.owner, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
-                new_release.save()
+                # avoid overwriting other fields
+                new_release.save(update_fields=["state", "failed", "summary", "exception"])
             if 'new_release' not in locals():
                 self.delete()
             raise DryccException(str(e)) from e
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 8ffc51247..ec4785888 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -1,13 +1,19 @@
 import logging
 
+from datetime import datetime
+from django.utils import timezone
 from django.conf import settings
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth import get_user_model
+from django.db.models import F, Func, Value, JSONField
 from api.tasks import run_pipeline
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
+
+from ..utils import DeployLock
+
 from .base import UuidAuditedModel
 from .appsettings import AppSettings
 
@@ -34,6 +40,7 @@ class Release(UuidAuditedModel):
     summary = models.TextField(blank=True, null=True)
     failed = models.BooleanField(default=False)
     exception = models.TextField(blank=True, null=True)
+    conditions = models.JSONField(default=list)
 
     config = models.ForeignKey('Config', on_delete=models.CASCADE)
     build = models.ForeignKey('Build', null=True, on_delete=models.CASCADE)
@@ -46,10 +53,6 @@ class Meta:
     def __str__(self):
         return "{0}-{1}".format(self.app.id, self.version_name)
 
-    @property
-    def deploying(self):
-        return self.build is not None and self.state == "created"
-
     @property
     def procfile_types(self):
         if self.build is not None:
@@ -77,6 +80,18 @@ def get_runners(self, procfile_types):
                     break
         return results
 
+    def add_condition(self, **kwargs):
+        if "created" not in kwargs:
+            kwargs["created"] = datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT)
+        type(self).objects.filter(pk=self.pk).update(
+            conditions=Func(
+                F("conditions"),
+                Value(["0"]),
+                Value(kwargs, JSONField()),
+                function="jsonb_insert",
+            )
+        )
+
     def get_deploy_image(self, container_type):
         """
         In the deploy phase of dryccfile
@@ -147,6 +162,12 @@ def get_port(self, procfile_type):
             procfile_type, {}).get(
                 'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
 
+    def deploy(self, procfile_types=None, force_deploy=False):
+        lock = DeployLock(self.app.pk)
+        if not lock.acquire(procfile_types, force=force_deploy):
+            raise DryccException('there is an executing pipeline, please wait or force deploy')
+        run_pipeline.delay(self, procfile_types, force_deploy)
+
     def previous(self):
         """
         Return the previous Release to this one.
@@ -191,7 +212,7 @@ def rollback(self, user, procfile_types=None, version=None):
                 summary="{} rolled back to v{}".format(user, version),
             )
             if self.build is not None:
-                run_pipeline.delay(new_release, procfile_types, force_deploy=True)
+                new_release.deploy(procfile_types, force_deploy=True)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
@@ -206,7 +227,8 @@ def rollback(self, user, procfile_types=None, version=None):
                     self.owner)
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
-                new_release.save()
+                # avoid overwriting other fields
+                new_release.save(update_fields=["state", "failed", "summary", "exception"])
             raise DryccException(str(e)) from e
 
     def cleanup_old(self, procfile_types=None):
@@ -378,5 +400,4 @@ def save(self, *args, **kwargs):  # noqa
                 else:
                     # There were no changes to this release
                     raise AlreadyExists("{} changed nothing - release stopped".format(self.owner))
-
         super(Release, self).save(*args, **kwargs)
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 1631546c4..4478b2c3d 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -279,7 +279,7 @@ def test_config_deploy_failure(self, mock_requests):
             app = App.objects.get(id=app_id)
             release = app.release_set.latest()
             self.assertEqual(release.failed, True)
-            self.assertEqual(release.exception, "error: Boom!")
+            self.assertEqual(release.conditions[0]['exception'], "Boom!")
 
     def test_invalid_config_keys(self, mock_requests):
         """Test that invalid config keys are rejected.
@@ -386,7 +386,7 @@ def test_config_failures(self, mock_requests):
             self.assertEqual(response.status_code, 201)
             self.assertEqual(app.release_set.latest().version, 4)
             self.assertEqual(app.release_set.latest().failed, True)
-            self.assertEqual(app.release_set.latest().exception, "error: Boom!")
+            self.assertEqual(app.release_set.latest().conditions[0]['exception'], "Boom!")
             self.assertEqual(app.release_set.filter(failed=False).latest().version, 3)
 
         # create a build to see that the new release is created with the last successful config
@@ -464,7 +464,7 @@ def test_unset_limits_error(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         # unset error
         body = {
             'limits': {
@@ -542,7 +542,7 @@ def test_set_config_limits_run(self, *args, **kwargs):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         body = {
             'values': json.dumps({'PORT': 5000}),
             'limits': {
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 340774948..6ff09293a 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -55,7 +55,7 @@ def create_app_with_domain_and_deploy(self):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         return app_id
 
     def change_certs_auto(self, app_id, enabled):
diff --git a/rootfs/api/tests/test_lock.py b/rootfs/api/tests/test_lock.py
new file mode 100644
index 000000000..85dc76858
--- /dev/null
+++ b/rootfs/api/tests/test_lock.py
@@ -0,0 +1,48 @@
+import time
+from api.tests import DryccTestCase
+from api.utils import CacheLock, DeployLock
+
+
+class TestLock(DryccTestCase):
+
+    def test_cache_lock(self):
+        key = f"test_key_1_{int(time.time())}"
+        lock1 = CacheLock(key)
+        lock2 = CacheLock(key)
+        timeout = 5
+        self.assertEqual(lock1.acquire(True, timeout), True)
+        self.assertEqual(lock2.acquire(False, timeout), False)
+        time.sleep(5)
+        start_time = time.time()
+        self.assertEqual(lock2.acquire(False, timeout), True)
+        self.assertEqual(lock1.acquire(True, timeout + 1), True)
+        self.assertTrue(time.time()-start_time > 5)
+        self.assertEqual(lock2.acquire(False, timeout), False)
+        lock1.release()
+        self.assertEqual(lock2.acquire(False, timeout), True)
+
+    def test_deploy_lock(self):
+        app_id = f"test_key_1_{int(time.time())}"
+        lock1 = DeployLock(app_id)
+        lock2 = DeployLock(app_id)
+        self.assertEqual(lock1.acquire(None), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        lock1.release(None)
+        self.assertEqual(lock2.acquire(["web", "task"]), True)
+        self.assertEqual(lock1.acquire(None), False)
+        self.assertEqual(lock1.acquire(["web"]), False)
+        self.assertEqual(lock1.acquire(["bing"]), True)
+        lock2.release(["web", "task"])
+        self.assertEqual(lock1.acquire(["web"]), True)
+        self.assertEqual(lock1.acquire(["task"]), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        self.assertEqual(lock2.acquire(None), False)
+        lock2.release(["web", "task", "bing"])
+        self.assertEqual(lock2.acquire(None), True)
+        lock2.release(None)
+        self.assertEqual(lock1.acquire(["web", "task"]), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        self.assertEqual(lock2.acquire(["web", "bing"], True), True)
+        self.assertEqual(lock1.acquire(["web", "bing", "task"]), False)
+        self.assertEqual(lock1.acquire(["web", "bing", "task"], True), True)
+        lock2.release(["web", "bing", "task"])
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index 6abf99687..f0581b4ad 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -278,16 +278,18 @@ def test_container_errors(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
         url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 'not_an_int'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
         self.assertEqual(app.structure, {'web': 1})
 
         body = {'invalid': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
         self.assertEqual(app.structure, {'web': 1})
 
     def test_container_str(self, mock_requests):
@@ -512,7 +514,7 @@ def test_command_good(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # use `start web` for backwards compatibility with buildpacks
         self.assertEqual(release.get_deploy_args('web'), [])
@@ -571,7 +573,7 @@ def test_run_command_good(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # create a run pod
         url = f"/v2/apps/{app_id}/run"
@@ -632,7 +634,7 @@ def test_run_not_fail_on_debug(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # create a run pod
         url = f"/v2/apps/{app_id}/run"
@@ -669,7 +671,7 @@ def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
             build=build
         )
         # deploy
-        app.pipeline(release)
+        release.deploy()
 
         # create a run pod
         url = f"/v2/apps/{app_id}/run"
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 8aa790fdb..b6eec3d66 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -105,6 +105,33 @@ def test_release(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.content)
         return release3
 
+    def test_conditions(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/builds"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker-test1': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = f'/v2/apps/{app_id}/releases/'
+        response = self.client.get(url)
+        self.assertEqual(len(response.data['results'][0]['conditions']), 1)
+
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.side_effect = Exception('Boom!')
+            url = f"/v2/apps/{app_id}/builds"
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            url = f'/v2/apps/{app_id}/releases/'
+            response = self.client.get(url)
+            self.assertEqual(response.data['results'][0]['conditions'][0]['exception'], 'Boom!')
+
     def test_get_image(self, mock_requests):
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/builds"
@@ -192,7 +219,8 @@ def test_response_data(self, mock_requests):
         response = self.client.get(url)
         for key in response.data.keys():
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'build', 'config',
-                                'summary', 'version', 'state', 'failed', 'exception'])
+                                'summary', 'version', 'state', 'failed', 'conditions',
+                                'exception'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index e732b8677..b9cc3a384 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -211,18 +211,74 @@ def acquire(self, blocking=True, timeout=120):
         value = None
         for _ in range(timeout):
             value = cache.get_or_set(self.key, self.value, timeout)
-            if blocking or value == self.value:
+            if not blocking or value == self.value:
                 break
             time.sleep(1)
         return value == self.value
 
     def release(self):
         value = cache.get(self.key, None)
-        if value == self.value:
+        if value != self.value:
             return
         cache.delete(self.key)
 
 
+class DeployLock(CacheLock):
+
+    def __init__(self, app_id):
+        self.procfile_types_key = f"app:deploy:procfile:types:{app_id}"
+        super(DeployLock, self).__init__(f"app:deploy:lock:{app_id}")
+
+    def locked(self, procfile_types):
+        value = cache.get(self.procfile_types_key, [])
+        # None will locks all
+        if procfile_types is None:
+            if value is None or len(value) > 0:
+                return True
+            elif len(value) == 0:
+                return False
+        else:
+            if value is None:
+                return True
+            for procfile_type in procfile_types:
+                if procfile_type in value:
+                    return True
+        return False
+
+    def acquire(self, procfile_types, force=False):
+        try:
+            if super(DeployLock, self).acquire():
+                value = cache.get(self.procfile_types_key, [])
+                # None will locks all
+                if not force and self.locked(procfile_types):
+                    return False
+                if procfile_types is None:
+                    value = None
+                else:
+                    value.extend(procfile_types)
+                    value = list(set(value))
+                cache.set(self.procfile_types_key, value, timeout=3600)
+                return True
+        finally:
+            super(DeployLock, self).release()
+        return False
+
+    def release(self, procfile_types):
+        try:
+            if super(DeployLock, self).acquire():
+                if procfile_types is None:
+                    cache.delete(self.procfile_types_key)
+                else:
+                    value = cache.get(self.procfile_types_key, [])
+                    if value is None:
+                        return
+                    for procfile_type in procfile_types:
+                        value.remove(procfile_type)
+                    cache.set(self.procfile_types_key, value)
+        finally:
+            super(DeployLock, self).release()
+
+
 class SyncIterToAsyncIter(object):
 
     def __init__(self, iter_content):
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index e7c2bcf36..5d1e6bc0b 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -26,7 +26,7 @@
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
-    delete_pod, run_pipeline
+    delete_pod
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
@@ -391,12 +391,9 @@ class ConfigViewSet(ReleasableViewSet):
     serializer_class = serializers.ConfigSerializer
 
     def post_save(self, config):
-        if not config.app.appsettings_set.latest().autorollback:
+        if not config.app.appsettings_set.latest().autodeploy:
             return
         latest_release = config.app.release_set.filter(failed=False).latest()
-        if latest_release.deploying:
-            config.delete()
-            raise DryccException('There is an executing pipeline, please wait')
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
@@ -406,9 +403,9 @@ def post_save(self, config):
                     for value in diff.values():
                         procfile_types.update(value.keys())
             # all_diff_fields changed, deploy all.
-            procfile_types = procfile_types if procfile_types else None
-            run_pipeline.delay(release, procfile_types, False)
-        except Exception as e:
+            procfile_types = list(procfile_types) if procfile_types else None
+            release.deploy(procfile_types, False)
+        except BaseException as e:
             config.delete()
             if isinstance(e, AlreadyExists):
                 raise
@@ -474,9 +471,6 @@ def restart(self, request, *args, **kwargs):
 
     def scale(self, request, **kwargs):
         app = self.get_app()
-        latest_release = app.release_set.filter(failed=False).latest()
-        if latest_release.deploying:
-            raise DryccException('There is an executing pipeline, please wait')
         scale_app.delay(app, request.user, request.data)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
@@ -636,13 +630,12 @@ def get_object(self, **kwargs):
 
     def deploy(self, request, **kwargs):
         """Deploy the latest release"""
-        release = self.get_app().release_set.latest()
-        if release.deploying:
-            raise DryccException('There is an executing pipeline, please wait')
+        latest_release = self.get_app().release_set.latest()
+        force_deploy = request.data.get("force", "false").lower() == "true"
         procfile_types = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        procfile_types = release.app.check_procfile_types(procfile_types)
-        run_pipeline.delay(release, procfile_types, False)
+        procfile_types = latest_release.app.check_procfile_types(procfile_types)
+        latest_release.deploy(procfile_types, force_deploy)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def rollback(self, request, **kwargs):
@@ -651,8 +644,6 @@ def rollback(self, request, **kwargs):
         previous release.
         """
         latest_release = self.get_app().release_set.filter(failed=False).latest()
-        if latest_release.deploying:
-            raise DryccException('There is an executing pipeline, please wait')
         procfile_types = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
         procfile_types = latest_release.app.check_procfile_types(procfile_types)

From 1402b5fdcd66898d16c62a414ed1575b904abcec Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 28 Aug 2024 23:19:28 +0800
Subject: [PATCH 066/200] feat(perms): refine the permission model

---
 rootfs/api/consumers.py                       |   4 +-
 .../0010_alter_certificate_options.py         |  17 --
 ...ploy.py => 0011_appsettings_autodeploy.py} |   2 +-
 .../migrations/0011_merge_20240815_0955.py    |  14 --
 ...ificate_app_release_conditions_and_more.py |  67 +++++++
 rootfs/api/models/app.py                      |  52 ++++-
 rootfs/api/models/base.py                     |  33 ----
 rootfs/api/models/certificate.py              |  12 +-
 rootfs/api/permissions.py                     |  17 +-
 rootfs/api/serializers/__init__.py            |   1 +
 rootfs/api/settings/production.py             |   2 +
 rootfs/api/tests/test_app.py                  |   8 +-
 rootfs/api/tests/test_certificate.py          |   9 +-
 .../api/tests/test_certificate_use_case_1.py  |   7 +-
 .../api/tests/test_certificate_use_case_2.py  |   5 +-
 .../api/tests/test_certificate_use_case_3.py  |   5 +-
 .../api/tests/test_certificate_use_case_4.py  |   5 +-
 .../api/tests/test_certificate_use_case_5.py  |   5 +-
 rootfs/api/tests/test_gateway.py              |   5 +-
 rootfs/api/tests/test_hooks.py                |   8 +-
 rootfs/api/tests/test_perm.py                 | 182 ++++++++++++------
 rootfs/api/urls.py                            |  43 +++--
 rootfs/api/views.py                           | 137 ++++++-------
 23 files changed, 368 insertions(+), 272 deletions(-)
 delete mode 100644 rootfs/api/migrations/0010_alter_certificate_options.py
 rename rootfs/api/migrations/{0012_appsettings_autodeploy.py => 0011_appsettings_autodeploy.py} (93%)
 delete mode 100644 rootfs/api/migrations/0011_merge_20240815_0955.py
 create mode 100644 rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index a04186050..e1552043d 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -20,7 +20,7 @@
 from channels.generic.websocket import AsyncWebsocketConsumer
 
 from .models.app import App
-from .permissions import has_object_permission
+from .permissions import has_app_permission
 
 
 class BaseAppConsumer(AsyncWebsocketConsumer):
@@ -35,7 +35,7 @@ def has_perm(self):
         if permission is None:
             try:
                 app = App.objects.get(id=self.id)
-                permission = has_object_permission(self.scope["user"], app, "GET")
+                permission = has_app_permission(self.scope["user"], app, "GET")
                 if permission[0]:
                     cache.set(key, permission, timeout=self.timeout)
             except App.DoesNotExist:
diff --git a/rootfs/api/migrations/0010_alter_certificate_options.py b/rootfs/api/migrations/0010_alter_certificate_options.py
deleted file mode 100644
index fd5e60f4b..000000000
--- a/rootfs/api/migrations/0010_alter_certificate_options.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# Generated by Django 4.2.15 on 2024-08-14 09:43
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0009_remove_appsettings_canaries_remove_release_canary_and_more'),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='certificate',
-            options={'ordering': ['name', 'common_name', 'expires'], 'permissions': (('use_cert', 'Can use cert'),)},
-        ),
-    ]
diff --git a/rootfs/api/migrations/0012_appsettings_autodeploy.py b/rootfs/api/migrations/0011_appsettings_autodeploy.py
similarity index 93%
rename from rootfs/api/migrations/0012_appsettings_autodeploy.py
rename to rootfs/api/migrations/0011_appsettings_autodeploy.py
index 5b5eb8d0d..5744324da 100644
--- a/rootfs/api/migrations/0012_appsettings_autodeploy.py
+++ b/rootfs/api/migrations/0011_appsettings_autodeploy.py
@@ -14,7 +14,7 @@ def migration_dryccfile(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0011_merge_20240815_0955'),
+        ('api', '0010_appsettings_autorollback'),
     ]
 
     operations = [
diff --git a/rootfs/api/migrations/0011_merge_20240815_0955.py b/rootfs/api/migrations/0011_merge_20240815_0955.py
deleted file mode 100644
index adeb0d7d6..000000000
--- a/rootfs/api/migrations/0011_merge_20240815_0955.py
+++ /dev/null
@@ -1,14 +0,0 @@
-# Generated by Django 4.2.15 on 2024-08-15 01:55
-
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0010_alter_certificate_options'),
-        ('api', '0010_appsettings_autorollback'),
-    ]
-
-    operations = [
-    ]
diff --git a/rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py b/rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py
new file mode 100644
index 000000000..fcda371e5
--- /dev/null
+++ b/rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py
@@ -0,0 +1,67 @@
+# Generated by Django 4.2.15 on 2024-08-30 00:53
+
+import api.utils
+from django.db import migrations, models
+import django.db.models.deletion
+from guardian.shortcuts import assign_perm, get_users_with_perms, remove_perm
+from api.models.app import App, VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
+from api.models.domain import Domain
+
+
+def migration_permission(apps, schema_editor):
+    for app in App.objects.all():
+        for user in get_users_with_perms(app):
+            remove_perm('use_app', user, app)
+            assign_perm(VIEW_APP_PERMISSION.codename, user, app)
+            assign_perm(CHANGE_APP_PERMISSION.codename, user, app)
+
+
+def migration_certificate(apps, schema_editor):
+    for domain in Domain.objects.all():
+        if domain.certificate:
+            if domain.certificate.app == None:
+                domain.certificate.app = domain.app
+                domain.certificate.save()
+            else:
+                certificate = domain.certificate
+                certificate.pk = None
+                certificate.app = domain.app
+                certificate.save()
+                domain.certificate = certificate
+                domain.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_appsettings_autodeploy'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='app',
+            options={'ordering': ['id'], 'verbose_name': 'Application'},
+        ),
+        migrations.AddField(
+            model_name='certificate',
+            name='app',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='api.app'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='conditions',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name='certificate',
+            name='name',
+            field=models.CharField(max_length=253, validators=[api.utils.validate_label]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='certificate',
+            unique_together={('app', 'name')},
+        ),
+        migrations.RunPython(migration_permission),
+        migrations.RunPython(migration_certificate),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 9fafcdf76..baa4a04cc 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -10,7 +10,7 @@
 import socket
 from contextlib import closing
 from urllib.parse import urljoin
-from collections import OrderedDict
+from collections import OrderedDict, namedtuple
 from datetime import datetime, timezone
 
 from docker import auth as docker_auth
@@ -31,12 +31,51 @@
 from .tls import TLS
 from .appsettings import AppSettings
 from .volume import Volume
-from .base import UuidAuditedModel, PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN, DEFAULT_HTTP_PORT, \
-    ObjectPolicy, object_policy_registry
+from .base import UuidAuditedModel, PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN, DEFAULT_HTTP_PORT
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
-app_policy = ObjectPolicy('id', 'use_app', 'Can use app')  # query field, policy, description
+AppPermission = namedtuple('AppPermission', ['shortname', 'codename', 'description'])
+VIEW_APP_PERMISSION = AppPermission("view", "view_app", "can view app")
+CHANGE_APP_PERMISSION = AppPermission("change", "change_app", "can change app")
+DELETE_APP_PERMISSION = AppPermission("delete", "delete_app", "can delete app")
+
+
+class AppPermissionRegistry(object):
+
+    def __init__(self):
+        self.tags = {}
+        self.permissions = set()
+
+    def get(self, q):
+        permissions = [
+            permission for permission in self.permissions
+            if q == permission.shortname or q == permission.codename
+        ]
+        permissions.extend([
+            permission for permission in self.permissions
+            if permission in self.tags and q in self.tags[permission]
+        ])
+        return permissions[0] if permissions else None
+
+    def register(self, permission, tags=None):
+        if tags:
+            self.tags[permission] = tags
+        self.permissions.add(permission)
+
+    @property
+    def codenames(self):
+        return [permission[1] for permission in self.permissions]
+
+    @property
+    def shortnames(self):
+        return [permission[0] for permission in self.permissions]
+
+
+app_permission_registry = AppPermissionRegistry()
+app_permission_registry.register(VIEW_APP_PERMISSION, ["GET", "HEAD", "OPTION"])
+app_permission_registry.register(CHANGE_APP_PERMISSION, ["POST", "PUT", "PATCH"])
+app_permission_registry.register(DELETE_APP_PERMISSION, ["DELETE"])
 
 
 # http://kubernetes.io/v1.1/docs/design/identifiers.html
@@ -79,7 +118,6 @@ class App(UuidAuditedModel):
 
     class Meta:
         verbose_name = 'Application'
-        permissions = (app_policy[1:], )
         ordering = ['id']
 
     def save(self, *args, **kwargs):
@@ -1168,7 +1206,3 @@ def _gather_app_settings(self, release, app_settings, procfile_type, replicas, v
             'pod_security_context': limit_plan.pod_security_context,
             'container_security_context': limit_plan.container_security_context,
         }
-
-
-# Register policy
-object_policy_registry.register(App, app_policy)
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 038de2970..87ae5353a 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -4,7 +4,6 @@
 import importlib
 from datetime import timedelta
 from functools import partial
-from collections import namedtuple
 from django.db import models
 from django.conf import settings
 from django.utils import timezone
@@ -39,38 +38,6 @@
 def get_anonymous_user_instance(user): return user(id=-1, username=settings.ANONYMOUS_USER_NAME)
 
 
-ObjectPolicy = namedtuple('ObjectPolicy', ['unique', 'codename', 'description'])
-
-
-class ObjectPolicyRegistry(object):
-
-    def __init__(self):
-        self.object_permission_policy_table = dict[type, ObjectPolicy]()
-
-    def all(self) -> list[type, ObjectPolicy]:
-        return self.object_permission_policy_table.items()
-
-    def get(self, query) -> tuple[type, ObjectPolicy]:
-        if isinstance(query, str):
-            for key, value in self.object_permission_policy_table.items():
-                if value.codename == query:
-                    return key, value
-        elif isinstance(query, models.Model):
-            query = type(query)
-            value = self.object_permission_policy_table.get(query)
-            return query if value else None, value
-        elif isinstance(query, type):
-            value = self.object_permission_policy_table.get(query)
-            return query if value else None, value
-        return None, None
-
-    def register(self, cls: type, object_policy: ObjectPolicy) -> None:
-        self.object_permission_policy_table[cls] = object_policy
-
-
-object_policy_registry = ObjectPolicyRegistry()
-
-
 class AuditedModel(models.Model):
     """Add created and updated fields to a model."""
 
diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index 7c002d3e3..d97d0c16f 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -15,12 +15,11 @@
 from api.utils import validate_label
 from api.exceptions import AlreadyExists, ServiceUnavailable
 from scheduler import KubeException
-from .base import AuditedModel, ObjectPolicy, object_policy_registry
+from .base import AuditedModel
 from .domain import Domain
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
-cert_policy = ObjectPolicy('name', 'use_cert', 'Can use cert')
 
 
 # Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
@@ -104,7 +103,8 @@ class Certificate(AuditedModel):
     Public and private key pair used to secure application traffic at the router.
     """
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
-    name = models.CharField(max_length=253, unique=True, validators=[validate_label])
+    app = models.ForeignKey('App', on_delete=models.CASCADE)
+    name = models.CharField(max_length=253, validators=[validate_label])
     # there is no upper limit on the size of an x.509 certificate
     certificate = models.TextField(validators=[validate_certificate])
     key = models.TextField(validators=[validate_private_key])
@@ -121,8 +121,8 @@ class Certificate(AuditedModel):
     subject = models.TextField(editable=False)
 
     class Meta:
-        permissions = (cert_policy[1:], )
         ordering = ['name', 'common_name', 'expires']
+        unique_together = ('app', 'name')
 
     @property
     def domains(self):
@@ -240,7 +240,3 @@ def detach(self, *args, **kwargs):
                 raise ServiceUnavailable(
                     "Could not delete certificate secret {} for application {}".format(
                         self.certname, namespace)) from e
-
-
-# Register policy
-object_policy_registry.register(Certificate, cert_policy)
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index 8ea9444ea..4f19a7213 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -5,7 +5,6 @@
 from api import manager
 from api.models import app
 from api.models import blocklist
-from api.models.base import object_policy_registry
 
 
 def get_app_status(app):
@@ -19,20 +18,20 @@ def get_app_status(app):
     return True, None
 
 
-def has_object_permission(user, obj, method):
+def has_app_permission(user, obj, method):
     obj = getattr(obj, 'app', obj)
-    object_policy = object_policy_registry.get(obj)[1]
     has_permission, message = False, f"{obj} object does not exist or does not have permission."
     if user.is_superuser:
         has_permission, message = True, None
     elif getattr(obj, "owner", None) == user:
         has_permission, message = True, None
-    elif user.is_staff or (object_policy and user.has_perm(object_policy.codename, obj)):
-        if method != 'DELETE':
+    elif user.is_staff:
+        has_permission, message = True, None
+    else:
+        permission = app.app_permission_registry.get(method)
+        if permission and user.has_perm(permission.codename, obj):
             has_permission, message = True, None
-        else:
-            has_permission, message = False, "{user} does not have permission to delete."
-    elif has_permission and isinstance(obj, app.App):
+    if has_permission and isinstance(obj, app.App):
         return get_app_status(obj)
     return has_permission, message
 
@@ -82,7 +81,7 @@ class IsObjectUser(permissions.BasePermission):
     an app-related model.
     """
     def has_object_permission(self, request, view, obj):
-        return has_object_permission(request.user, obj, request.method)[0]
+        return has_app_permission(request.user, obj, request.method)[0]
 
 
 class IsAdmin(permissions.BasePermission):
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index eb13a8f7f..a3e018fd6 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -506,6 +506,7 @@ def validate_target_port(cls, value):
 class CertificateSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.certificate.Certificate` model."""
 
+    app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     owner = serializers.ReadOnlyField(source='owner.username')
     domains = serializers.ReadOnlyField()
     san = serializers.ListField(
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 83c27d57d..f306a57d5 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -405,6 +405,8 @@
 
 APP_URL_REGEX = '[a-z0-9-]+'
 
+DOMAIN_URL_REGEX = r'\**\.?[-\._\w]+'
+
 # Oauth settings
 
 DRYCC_PASSPORT_URL = os.environ.get('DRYCC_PASSPORT_URL', 'https://127.0.0.1:8000')
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 4d880266f..7221e8164 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -15,14 +15,13 @@
 from django.core.cache import cache
 from django.test.utils import override_settings
 
-from api.models.app import App
+from api.models.app import App, app_permission_registry
 from api.models.base import PROCFILE_TYPE_WEB
 from api.models.config import Config
 from scheduler import KubeException, KubeHTTPException
 
 from api.exceptions import DryccException
 from api.tests import adapter, DryccTestCase
-from api.models.base import object_policy_registry
 import requests_mock
 
 User = get_user_model()
@@ -361,10 +360,9 @@ def test_app_transfer(self, mock_requests):
         # Collaborators can't transfer
         body = {
             'username': owner.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app.id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
-        response = self.client.post('/v2/perms/rules/', body)
+        response = self.client.post(f'/v2/apps/{app.id}/perms/', body)
         self.assertEqual(response.status_code, 201, response.data)
 
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
diff --git a/rootfs/api/tests/test_certificate.py b/rootfs/api/tests/test_certificate.py
index db6b11c3d..c7efd8f0f 100644
--- a/rootfs/api/tests/test_certificate.py
+++ b/rootfs/api/tests/test_certificate.py
@@ -2,6 +2,7 @@
 from django.core.cache import cache
 from django.core.exceptions import SuspiciousOperation
 
+from api.models.app import App
 from api.models.certificate import Certificate
 from api.tests import TEST_ROOT, DryccTestCase
 
@@ -18,8 +19,8 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-        self.url = '/v2/certs'
+        self.app = App.objects.create(owner=self.user, id='test-app-use-case')
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domain = 'autotest.example.com'
 
         with open('{}/certs/{}.key'.format(TEST_ROOT, self.domain)) as f:
@@ -144,12 +145,13 @@ def test_delete_certificate(self):
         """Destroying a certificate should generate a 204 response"""
         Certificate.objects.create(
             name='random-test-cert',
+            app=self.app,
             owner=self.user,
             common_name='autotest.example.com',
             certificate=self.cert,
             key=self.key
         )
-        url = '/v2/certs/random-test-cert'
+        url = f'/v2/apps/{self.app.id}/certs/random-test-cert'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
 
@@ -173,6 +175,7 @@ def test_load_invalid_cert(self):
         with self.assertRaises(SuspiciousOperation):
             Certificate.objects.create(
                 owner=self.user,
+                app=self.app,
                 name='random-test-cert',
                 certificate='i am bad data',
                 key='i am bad data as well'
diff --git a/rootfs/api/tests/test_certificate_use_case_1.py b/rootfs/api/tests/test_certificate_use_case_1.py
index 1ec9a361f..725bba70c 100644
--- a/rootfs/api/tests/test_certificate_use_case_1.py
+++ b/rootfs/api/tests/test_certificate_use_case_1.py
@@ -24,8 +24,8 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
         self.app = App.objects.create(owner=self.user, id='test-app-use-case-1')
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domain = Domain.objects.create(
             owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB)
         self.name = 'foo-com'  # certificate name
@@ -148,12 +148,13 @@ def test_delete_certificate(self):
         """Destroying a certificate should generate a 204 response"""
         Certificate.objects.create(
             owner=self.user,
+            app=self.app,
             name=self.name,
             certificate=self.cert,
             key=self.key
         )
 
-        url = '/v2/certs/{}'.format(self.name)
+        url = f'/v2/apps/{self.app.id}/certs/{self.name}/'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
 
@@ -190,7 +191,7 @@ def test_delete_certificate_with_attached_domain(self):
         self.assertEqual(domain.certificate.name, self.name)
 
         # Delete certificate
-        url = '/v2/certs/{}'.format(self.name)
+        url = f'/v2/apps/{self.app.id}/certs/{self.name}/'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
 
diff --git a/rootfs/api/tests/test_certificate_use_case_2.py b/rootfs/api/tests/test_certificate_use_case_2.py
index 907bceb1c..646aaa47b 100644
--- a/rootfs/api/tests/test_certificate_use_case_2.py
+++ b/rootfs/api/tests/test_certificate_use_case_2.py
@@ -24,8 +24,8 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
         self.app = App.objects.create(owner=self.user, id='test-app-use-case-2')
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             'foo.com': Domain.objects.create(
                 owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB),
@@ -132,11 +132,12 @@ def test_delete_certificate(self):
         for domain, certificate in self.certificates.items():
             Certificate.objects.create(
                 name=certificate['name'],
+                app=self.app,
                 owner=self.user,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_certificate_use_case_3.py b/rootfs/api/tests/test_certificate_use_case_3.py
index 4caff2aee..4163dcd92 100644
--- a/rootfs/api/tests/test_certificate_use_case_3.py
+++ b/rootfs/api/tests/test_certificate_use_case_3.py
@@ -24,8 +24,8 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
         self.app = App.objects.create(owner=self.user, id='test-app-use-case-3')
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             'foo.com': Domain.objects.create(
                 owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB),
@@ -134,11 +134,12 @@ def test_delete_certificate(self):
         for domain, certificate in self.certificates.items():
             Certificate.objects.create(
                 name=certificate['name'],
+                app=self.app,
                 owner=self.user,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_certificate_use_case_4.py b/rootfs/api/tests/test_certificate_use_case_4.py
index 7f829d6fb..4ed65b506 100644
--- a/rootfs/api/tests/test_certificate_use_case_4.py
+++ b/rootfs/api/tests/test_certificate_use_case_4.py
@@ -24,8 +24,8 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
         self.app = App.objects.create(owner=self.user, id='test-app-use-case-3')
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             '*.foo.com': Domain.objects.create(
                 owner=self.user, app=self.app, domain='*.foo.com',
@@ -219,10 +219,11 @@ def test_delete_certificate(self):
             Certificate.objects.create(
                 name=certificate['name'],
                 owner=self.user,
+                app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_certificate_use_case_5.py b/rootfs/api/tests/test_certificate_use_case_5.py
index 0a352241e..a69e2809c 100644
--- a/rootfs/api/tests/test_certificate_use_case_5.py
+++ b/rootfs/api/tests/test_certificate_use_case_5.py
@@ -24,8 +24,8 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
         self.app = App.objects.create(owner=self.user, id='test-app-use-case-5')
+        self.url = f'/v2/apps/{self.app.id}/certs'
         # Done out of scope as it gets the same cert as the wildcard
         Domain.objects.create(
             owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB)
@@ -165,12 +165,13 @@ def test_delete_certificate(self):
             Certificate.objects.create(
                 name=certificate['name'],
                 owner=self.user,
+                app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
 
             # Remove certificate
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 6ff09293a..56fee99ce 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -72,7 +72,7 @@ def create_gateway(self, app_id, name, port, protocol):
         self.assertEqual(response.status_code, 201)
 
     def create_tls_domain(self, app_id):
-        cert_url = '/v2/certs'
+        cert_url = f'/v2/apps/{app_id}/certs'
         secret_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(23))
         domain = 'autotest.example.com'
 
@@ -96,7 +96,6 @@ def create_tls_domain(self, app_id):
             {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB}
         )
         self.assertEqual(response.status_code, 201)
-
         response = self.client.post(
             '{}/{}/domain/'.format(cert_url, secret_name),
             {'domain': domain}
@@ -294,7 +293,7 @@ def test_remove_tls(self):
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
         self.assertEqual(len(response.data["results"][0]["listeners"]), 1)
         response = self.client.delete(
-            '{}/{}/domain/{}/'.format('/v2/certs', secret_name, domain)
+            '{}/{}/domain/{}/'.format(f'/v2/apps/{app_id}/certs', secret_name, domain)
         )
         self.assertEqual(response.status_code, 204)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index 81923dd2d..fc85ed369 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -8,8 +8,7 @@
 from django.core.cache import cache
 
 from api.tests import adapter, DryccTransactionTestCase
-from api.models.app import App
-from api.models.base import object_policy_registry
+from api.models.app import app_permission_registry
 import requests_mock
 
 User = get_user_model()
@@ -74,10 +73,9 @@ def test_key_hook(self, mock_requests):
         # give user permission to app
         body = {
             'username': str(self.user),
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
-        url = "/v2/perms/rules"
+        url = f'/v2/apps/{app_id}/perms/'
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
diff --git a/rootfs/api/tests/test_perm.py b/rootfs/api/tests/test_perm.py
index 80cf8206f..21055e2cf 100644
--- a/rootfs/api/tests/test_perm.py
+++ b/rootfs/api/tests/test_perm.py
@@ -1,8 +1,8 @@
 from django.test import tag
 from django.contrib.auth import get_user_model
 from api.tests import DryccTestCase
-from api.models.base import object_policy_registry
-from api.models.app import App
+from api.models.app import App, app_permission_registry, \
+    VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
 
 User = get_user_model()
 
@@ -23,18 +23,6 @@ def setUp(self):
         self.user3 = User.objects.get(username='autotest-3')
         self.token3 = self.get_or_create_token(self.user3)
 
-    @tag('auth')
-    def test_codes(self):
-        response = self.client.get('/v2/perms/codes/')
-        expect = {
-            'results': [
-                {'codename': 'use_app', 'description': 'Can use app'},
-                {'codename': 'use_cert', 'description': 'Can use cert'}
-            ],
-            'count': 2
-        }
-        self.assertEqual(expect, response.json())
-
     @tag('auth')
     def test_create(self):
         # check that user 1 sees her lone app and user 2's app
@@ -57,19 +45,14 @@ def test_create(self):
             self.assertEqual(response.data['detail'],
                              'You do not have permission to perform this action.', msg=msg)
 
-        url = "/v2/perms/rules/"
-        for cls, object_policy in object_policy_registry.all():
-            if not cls.objects.exists():
-                continue
-            for obj in cls.objects.filter(owner=self.user):
-                body = {
-                    'username': self.user2.username,
-                    'codename': object_policy.codename,
-                    'uniqueid': getattr(obj, object_policy.unique)
-                }
-                self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-                response = self.client.post(url, body)
-                self.assertEqual(response.status_code, 201, response.data)
+        for app in App.objects.filter(owner=self.user):
+            body = {
+                'username': self.user2.username,
+                'permissions': ','.join(app_permission_registry.shortnames),
+            }
+            self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+            response = self.client.post(f'/v2/apps/{app.id}/perms/', body)
+            self.assertEqual(response.status_code, 201, response.data)
 
         # check that user 2 can see the app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
@@ -93,13 +76,12 @@ def test_create_errors(self):
 
         body = {
             'username': self.user2.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
 
         # check that user 2 can't create a permission
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post("/v2/perms/rules/", body)
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
         self.assertEqual(response.status_code, 403)
 
     @tag('auth')
@@ -109,10 +91,10 @@ def test_delete(self):
         app_id = response.data['results'][0]['id']
         body = {
             'username': self.user2.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(
+                [VIEW_APP_PERMISSION.shortname, CHANGE_APP_PERMISSION.shortname]),
         }
-        response = self.client.post("/v2/perms/rules/", body)
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # check that user 2 can see the app as well as his own
@@ -123,10 +105,9 @@ def test_delete(self):
 
         # delete permission to user 1's app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.get("/v2/perms/rules/")
-        url = "/v2/perms/rules/{}/".format(response.json()["results"][0]["id"])
+        url = f"/v2/apps/{app_id}/perms/{self.user2.username}/"
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204, response.data)
+        self.assertEqual(response.status_code, 204, response)
         self.assertIsNone(response.data)
 
         # check that user 2 can only see his app
@@ -137,7 +118,7 @@ def test_delete(self):
         # delete permission to user 1's app again, expecting an error
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 404)
+        self.assertEqual(response.status_code, 204)
 
     @tag('auth')
     def test_list(self):
@@ -150,14 +131,13 @@ def test_list(self):
         # create a new object permission
         body = {
             'username': self.user2.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
-        response = self.client.post("/v2/perms/rules/", body)
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # list perms on the app
-        response = self.client.get("/v2/perms/rules/")
+        response = self.client.get(f"/v2/apps/{app_id}/perms/")
         self.assertEqual(response.data["count"], 1)
 
     @tag('auth')
@@ -172,9 +152,8 @@ def test_list_errors(self):
         response = self.client.get('/v2/apps')
         # login as user 2, list perms on the app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get("/v2/perms/rules/")
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data, {'results': [], 'count': 0})
+        response = self.client.get(f"/v2/apps/{self.user2.id}/perms/")
+        self.assertEqual(response.status_code, 404)
 
     @tag('auth')
     def test_unauthorized_user_cannot_modify_perms(self):
@@ -191,15 +170,14 @@ def test_unauthorized_user_cannot_modify_perms(self):
 
         body = {
             'username': self.user2.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post('/v2/perms/rules/', body)
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
         self.assertEqual(response.status_code, 403)
 
     @tag('auth')
-    def test_collaborator_cannot_share(self):
+    def test_collaborator_share(self):
         """
         An collaborator should not be able to modify the app's permissions.
         """
@@ -211,23 +189,21 @@ def test_collaborator_cannot_share(self):
         # Share app with collaborator
         body = {
             'username': collab.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
-        url = '/v2/perms/rules/'
+        url = f'/v2/apps/{app_id}/perms/'
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        # Collaborator should fail to share app
+        # Collaborator can share app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
         body = {
             'username': self.user3.username,
-            'codename': object_policy_registry.get(App)[1].codename,
-            'uniqueid': app_id,
+            'permissions': ','.join(app_permission_registry.shortnames),
         }
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 201)
 
         # Collaborator can list
         response = self.client.get(url)
@@ -241,15 +217,103 @@ def test_collaborator_cannot_share(self):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token3)
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 1, response.data)
+        self.assertEqual(response.data['count'], 2, response.data)
 
         # Collaborator cannot delete other collaborator
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
-        url += "{}/".format(response.data["results"][0]["id"])
+        url += "{}/".format(self.user3.username)
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 204)
+        url = f'/v2/apps/{app_id}/perms/'
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['count'], 1, response.data)
 
         # Collaborator can delete themselves
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token3)
+        url += "{}/".format(self.user2.username)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204)
+        url = f'/v2/apps/{app_id}/perms/'
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['count'], 0, response.data)
+
+    @tag('auth')
+    def test_each_permission(self):
+        # check that user 1 sees her lone app and user 2's app
+        response = self.client.get('/v2/apps')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 2)
+        app_id = response.data['results'][0]['id']
+
+        # list no perms on the app
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.get(f"/v2/apps/{app_id}/perms/")
+        self.assertEqual(response.status_code, 403)
+
+        # create a new object permission
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        body = {
+            'username': self.user2.username,
+            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
+        }
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.get(f"/v2/apps/{app_id}/perms/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 1, response.data)
+
+        # test no change permission
+        body = {
+            'username': self.user2.username,
+            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
+        }
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
+        self.assertEqual(response.status_code, 403, response.data)
+
+        # test add change permission
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        body = {
+            'username': self.user2.username,
+            'permissions': ','.join(
+                [VIEW_APP_PERMISSION.shortname, CHANGE_APP_PERMISSION.shortname]),
+        }
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
+        self.assertEqual(response.status_code, 201)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        body = {
+            'username': self.user2.username,
+            'permissions': ','.join(
+                [VIEW_APP_PERMISSION.shortname, CHANGE_APP_PERMISSION.shortname]),
+        }
+        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
+        self.assertEqual(response.status_code, 201)
+
+        # test no delete permission
+        response = self.client.delete(f'/v2/apps/{app_id}/perms/{self.user2.username}/')
+        self.assertEqual(response.status_code, 403)
+
+        # test update permisssion
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        body = {
+            'username': self.user2.username,
+            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
+        }
+        response = self.client.put(f'/v2/apps/{app_id}/perms/{self.user2.username}/', body)
+        self.assertEqual(response.status_code, 204)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+
+        # test not update permission
+        body = {
+            'username': self.user2.username,
+            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
+        }
+        response = self.client.put(f'/v2/apps/{app_id}/perms/{self.user2.username}/', body)
+        self.assertEqual(response.status_code, 403)
+
+        # has view permission
+        response = self.client.get(f"/v2/apps/{app_id}/perms/")
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data['count'], 1, response.data)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index e894a3116..5948043d3 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -70,15 +70,16 @@
         views.PtypesViewSet.as_view({'get': 'list'})),
     # describe ptypes
     re_path(
-        r"^apps/(?P<id>{})/ptypes/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),  # noqa
+        r"^apps/(?P<id>{})/ptypes/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),
         views.PtypesViewSet.as_view({'get': 'describe'})),
     # list events
     re_path(
-        r"^apps/(?P<id>{})/events/?$".format(settings.APP_URL_REGEX),  # noqa
+        r"^apps/(?P<id>{})/events/?$".format(settings.APP_URL_REGEX),
         views.EventViewSet.as_view({'get': 'list'})),
     # application domains
     re_path(
-        r"^apps/(?P<id>{})/domains/(?P<domain>\**\.?[-\._\w]+)/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/domains/(?P<domain>{})/?$".format(
+            settings.APP_URL_REGEX, settings.DOMAIN_URL_REGEX),
         views.DomainViewSet.as_view({'delete': 'destroy'})),
     re_path(
         r"^apps/(?P<id>{})/domains/?$".format(settings.APP_URL_REGEX),
@@ -135,6 +136,20 @@
     re_path(
         r"^apps/(?P<id>{})/resources/(?P<name>[-_\w]+)/binding/?$".format(settings.APP_URL_REGEX),
         views.AppResourceBindingViewSet.as_view({'patch': 'binding'})),
+    # certificates
+    re_path(
+        r'^apps/(?P<id>{})/certs/(?P<name>[-_*.\w]+)/domain/(?P<domain>{})?/?$'.format(
+            settings.APP_URL_REGEX, settings.DOMAIN_URL_REGEX),
+        views.CertificateViewSet.as_view({'delete': 'detach', 'post': 'attach'})),
+    re_path(
+        r'^apps/(?P<id>{})/certs/(?P<name>[-_*.\w]+)/?$'.format(settings.APP_URL_REGEX),
+        views.CertificateViewSet.as_view({
+            'get': 'retrieve',
+            'delete': 'destroy'
+        })),
+    re_path(
+        r'^apps/(?P<id>{})/certs/?$'.format(settings.APP_URL_REGEX),
+        views.CertificateViewSet.as_view({'get': 'list', 'post': 'create'})),
     # apps base endpoint
     re_path(
         r"^apps/(?P<id>{})/?$".format(settings.APP_URL_REGEX),
@@ -151,7 +166,7 @@
         views.KeyViewSet.as_view({'get': 'list', 'post': 'create'})),
     # hooks
     re_path(
-        r'^hooks/keys/(?P<id>{})/(?P<username>[-_\w]+)/?$'.format(settings.APP_URL_REGEX),
+        r'^hooks/keys/(?P<id>{})/(?P<username>[\w.@+-]+)/?$'.format(settings.APP_URL_REGEX),
         views.KeyHookViewSet.as_view({'get': 'users'})),
     re_path(
         r'^hooks/keys/(?P<id>{})/?$'.format(settings.APP_URL_REGEX),
@@ -169,19 +184,6 @@
     re_path(
         r'^auth/whoami/?$',
         views.UserManagementViewSet.as_view({'get': 'list'})),
-    # certificates
-    re_path(
-        r'^certs/(?P<name>[-_*.\w]+)/domain/(?P<domain>\**\.?[-\._\w]+)?/?$',
-        views.CertificateViewSet.as_view({'delete': 'detach', 'post': 'attach'})),
-    re_path(
-        r'^certs/(?P<name>[-_*.\w]+)/?$',
-        views.CertificateViewSet.as_view({
-            'get': 'retrieve',
-            'delete': 'destroy'
-        })),
-    re_path(
-        r'^certs/?$',
-        views.CertificateViewSet.as_view({'get': 'list', 'post': 'create'})),
     # gateways
     re_path(
         r"^apps/(?P<id>{})/gateways/?$".format(settings.APP_URL_REGEX),
@@ -223,11 +225,12 @@
         r'^manager/(?P<type>[\w.@+-]+)s/(?P<id>{})/unblock/?$'.format(settings.APP_URL_REGEX),
         views.WorkflowManagerViewset.as_view({'delete': 'unblock'})),
     # user perms
-    re_path(r"^perms/codes/?$", views.UserPermViewSet.as_view({'get': 'codes'})),
     re_path(
-        r"^perms/rules/?$", views.UserPermViewSet.as_view({'get': 'list', 'post': 'create'})),
+        r"^apps/(?P<id>{})/perms/?$".format(settings.APP_URL_REGEX),
+        views.AppPermViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
-        r"^perms/rules/(?P<id>[-_\w]+)/?$", views.UserPermViewSet.as_view({'delete': 'destroy'})),
+        r"^apps/(?P<id>{})/perms/(?P<username>[\w.@+-]+)/?$".format(settings.APP_URL_REGEX),
+        views.AppPermViewSet.as_view({'put': 'update', 'delete': 'destroy'})),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
     re_path(r"^tokens/(?P<pk>[-_\w]+)/?$", views.TokenViewSet.as_view({'delete': 'destroy'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 5d1e6bc0b..fa95edbb9 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -13,7 +13,7 @@
 from django.contrib.auth import get_user_model
 from django.shortcuts import get_object_or_404, redirect
 from guardian.shortcuts import assign_perm, get_objects_for_user, \
-    get_users_with_perms, remove_perm, UserObjectPermission
+    get_users_with_perms, get_user_perms, remove_perm
 from django.views.generic import View
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
@@ -294,7 +294,8 @@ def list(self, request, *args, **kwargs):
         interact with.
         """
         queryset = super(AppViewSet, self).get_queryset(**kwargs) | \
-            get_objects_for_user(self.request.user, f'api.{models.app.app_policy[1]}')
+            get_objects_for_user(
+                self.request.user, f'api.{models.app.VIEW_APP_PERMISSION.codename}')
         instance = self.filter_queryset(queryset)
         page = self.paginate_queryset(instance)
         if page is not None:
@@ -568,16 +569,11 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class CertificateViewSet(BaseDryccViewSet):
+class CertificateViewSet(AppResourceViewSet):
     """A viewset for interacting with Certificate objects."""
     model = models.certificate.Certificate
     serializer_class = serializers.CertificateSerializer
 
-    def get_queryset(self, **kwargs):
-        return self.model.objects.filter(owner=self.request.user, **kwargs) | \
-            get_objects_for_user(
-                self.request.user, f'api.{models.certificate.cert_policy[1]}')
-
     def get_object(self, **kwargs):
         """Retrieve domain certificate by its name"""
         qs = self.get_queryset(**kwargs)
@@ -603,13 +599,6 @@ def detach(self, request, *args, **kwargs):
             raise
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def destroy(self, *args, **kwargs):
-        resource = self.get_object()
-        if self.request.user != resource.owner and not self.request.user.is_superuser:
-            return Response(status=status.HTTP_403_FORBIDDEN)
-        resource.delete()
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
 
 class KeyViewSet(BaseDryccViewSet):
     """A viewset for interacting with Key objects."""
@@ -676,7 +665,8 @@ def public_key(self, request, *args, **kwargs):
         key = get_object_or_404(models.key.Key, fingerprint=fingerprint)
 
         queryset = models.app.App.objects.all() | \
-            get_objects_for_user(self.request.user, f'api.{models.app.app_policy[1]}')
+            get_objects_for_user(
+                self.request.user, f'api.{models.app.VIEW_APP_PERMISSION.codename}')
         items = self.filter_queryset(queryset)
 
         apps = []
@@ -693,7 +683,7 @@ def public_key(self, request, *args, **kwargs):
     def app(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
         usernames = [u.id for u in get_users_with_perms(app)
-                     if u.has_perm(f"api.{models.app.app_policy[1]}", app)]
+                     if u.has_perm(f"api.{models.app.VIEW_APP_PERMISSION.codename}", app)]
 
         data = {}
         result = models.key.Key.objects \
@@ -716,8 +706,7 @@ def users(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
         request.user = get_object_or_404(User, username=kwargs['username'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_object_permission(
-            request.user, app, request.method)
+        has_permission, message = permissions.has_app_permission(request.user, app, request.method)
         if not has_permission:
             return Response(message, status=status.HTTP_403_FORBIDDEN)
 
@@ -747,7 +736,7 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         self.user = request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_object_permission(self.user, app, request.method)
+        has_permission, message = permissions.has_app_permission(self.user, app, request.method)
         if not has_permission:
             return Response(message, status=status.HTTP_403_FORBIDDEN)
         request.data['app'] = app
@@ -774,8 +763,7 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_object_permission(
-            request.user, app, request.method)
+        has_permission, message = permissions.has_app_permission(request.user, app, request.method)
         if not has_permission:
             return Response(message, status=status.HTTP_403_FORBIDDEN)
         config = app.release_set.filter(failed=False).latest().config
@@ -783,70 +771,73 @@ def create(self, request, *args, **kwargs):
         return Response(serializer.data, status=status.HTTP_200_OK)
 
 
-class UserPermViewSet(BaseDryccViewSet):
+class AppPermViewSet(AppResourceViewSet):
     """RESTful views for sharing apps with collaborators."""
 
-    def codes(self, request, **kwargs):
-        results = []
-        for _, object_policy in permissions.object_policy_registry.all():
-            results.append({
-                "codename": object_policy.codename,
-                "description": object_policy.description,
-            })
-        # fake out pagination for now
-        pagination = {'results': results, 'count': len(results)}
-        return Response(data=pagination)
-
     def list(self, request, **kwargs):
-        cls, object_policy = permissions.object_policy_registry.get(
-            request.query_params.get('codename', ''))
-        if object_policy:
-            object_policies = [(cls, object_policy), ]
-        else:
-            object_policies = permissions.object_policy_registry.all()
-        results = []
-        for cls, object_policy in object_policies:
-            user_object_perms = UserObjectPermission.objects.filter(
-                object_pk__in=[obj.pk for obj in cls.objects.filter(owner=request.user)],
-                permission__codename=object_policy.codename
-            )
-            user_object_perms |= UserObjectPermission.objects.filter(
-                user=request.user, permission__codename=object_policy.codename)
-
-            for user_object_perm in user_object_perms:
-                results.append({
-                    "id": user_object_perm.pk,
-                    "codename": user_object_perm.permission.codename,
-                    "uniqueid": getattr(user_object_perm.content_object, object_policy.unique),
-                    "username": user_object_perm.user.username,
-                })
+        app = self.get_app()
+        results = [
+            {
+                "app": app.id,
+                "username": user.username,
+                "permissions": [
+                    models.app.app_permission_registry.get(codename).shortname
+                    for codename in get_user_perms(user, app)
+                ],
+            }
+            for user in get_users_with_perms(app)
+        ]
         # fake out pagination for now
         pagination = {'results': results, 'count': len(results)}
         return Response(data=pagination)
 
     def create(self, request, **kwargs):
+        app = self.get_app()
         username = request.data.get('username')
-        codename = request.data.get('codename')
-        uniqueid = request.data.get('uniqueid')
-        cls, object_policy = permissions.object_policy_registry.get(codename)
-        if not cls or not object_policy:
-            return Response(f"User {codename} not found", status=status.HTTP_404_NOT_FOUND)
-        obj = get_object_or_404(cls, **{object_policy.unique: uniqueid})
-        if not permissions.IsOwnerOrAdmin().has_object_permission(request, self, obj):
-            return Response(status=status.HTTP_403_FORBIDDEN)
+        shortnames = set([perm for perm in request.data.get("permissions", "").split(",") if perm])
+        all_shortnames = models.app.app_permission_registry.shortnames
+        if not shortnames or not shortnames.issubset(all_shortnames):
+            msg = "The permissions field is required and has a value range of: {}".format(
+                ",".join(all_shortnames)
+            )
+            return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
         user = get_object_or_404(User, username=username)
-        assign_perm(object_policy.codename, user, obj)
-        getattr(obj, "log", logger.info)("User {} was granted access to {}".format(user, obj))
+        for shortname in shortnames:
+            permission = models.app.app_permission_registry.get(shortname)
+            if permission:
+                assign_perm(permission.codename, user, app)
+        app.log("User {} was granted access to {}".format(user, app))
         return Response(status=status.HTTP_201_CREATED)
 
+    def update(self, request, **kwargs):
+        app = self.get_app()
+        user = get_object_or_404(User, username=kwargs['username'])
+        shortnames = set([perm for perm in request.data.get("permissions", "").split(",") if perm])
+        all_shortnames = models.app.app_permission_registry.shortnames
+        if not shortnames or not shortnames.issubset(all_shortnames):
+            msg = "The permissions field is required and has a value range of: {}".format(
+                ",".join(all_shortnames)
+            )
+            return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
+        for shortname in shortnames.symmetric_difference([
+                models.app.app_permission_registry.get(codename).shortname
+                for codename in get_user_perms(user, app)]):
+            permission = models.app.app_permission_registry.get(shortname)
+            if permission:
+                if shortname in shortnames:
+                    assign_perm(permission.codename, user, app)
+                else:
+                    remove_perm(permission.codename, user, app)
+        app.log("User {} was revoked access to {}".format(user, app))
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
     def destroy(self, request, **kwargs):
-        user_object_permission = get_object_or_404(UserObjectPermission, id=self.kwargs['id'])
-        obj = user_object_permission.content_object
-        if not (obj.owner == request.user or user_object_permission.user == request.user):
-            return Response(status=status.HTTP_403_FORBIDDEN)
-        perm_name = f"api.{user_object_permission.permission.codename}"
-        remove_perm(perm_name, user_object_permission.user, obj)
-        getattr(obj, "log", logger.info)("User {} was revoked access to {}".format(user_object_permission.user, obj))  # noqa
+        app = self.get_app()
+        username = kwargs['username']
+        user = get_object_or_404(User, username=username)
+        for codename in get_user_perms(user, app):
+            remove_perm(codename, user, app)
+        app.log("User {} was revoked access to {}".format(user, app))
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 

From 5d67d65b5996fca1f99296341289855cad4054e5 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 30 Aug 2024 18:07:25 +0800
Subject: [PATCH 067/200] chore(controller): pod failed do not rollback (#159)

---
 rootfs/scheduler/resources/pod.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index b9695c38b..941f1bad6 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -523,6 +523,7 @@ def _handle_pod_errors(self, pod, reason, message):
             'ImagePullBackOff',
             'RegistryUnavailable',
             'ErrImageInspect',
+            "CreateContainerError",
         ]
         # Image event reason mapping
         event_errors = {
@@ -537,7 +538,7 @@ def _handle_pod_errors(self, pod, reason, message):
 
         # Nicer error than from the event
         # Often this gets to ImageBullBackOff before we can introspect tho
-        if reason == 'ErrImagePull':
+        if reason in ['ErrImagePull', 'CreateContainerError']:
             raise KubeException(message)
 
         # collect all error messages of worth

From 814912264c689c3ccbc1750c0edf4f701efd2943 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 30 Aug 2024 21:42:51 +0800
Subject: [PATCH 068/200] chore(controller): unified naming convention for
 ptype

---
 rootfs/api/admissions.py                      |  18 +-
 ...ame_procfile_type_domain_ptype_and_more.py |  58 ++++++
 ...migration_permissions_and_certificates.py} |  32 +--
 rootfs/api/models/app.py                      | 186 +++++++++---------
 rootfs/api/models/base.py                     |   8 +-
 rootfs/api/models/build.py                    |  26 +--
 rootfs/api/models/config.py                   |  36 ++--
 rootfs/api/models/domain.py                   |   4 +-
 rootfs/api/models/gateway.py                  |  10 +-
 rootfs/api/models/release.py                  |  76 +++----
 rootfs/api/models/service.py                  |  16 +-
 rootfs/api/monitor.py                         |  16 +-
 rootfs/api/serializers/__init__.py            |  60 +++---
 rootfs/api/signals.py                         |   2 +-
 rootfs/api/tests/__init__.py                  |  10 +-
 rootfs/api/tests/test_admissions.py           |   4 +-
 rootfs/api/tests/test_app.py                  |   8 +-
 rootfs/api/tests/test_build.py                |   6 +-
 .../api/tests/test_certificate_use_case_1.py  |   4 +-
 .../api/tests/test_certificate_use_case_2.py  |   6 +-
 .../api/tests/test_certificate_use_case_3.py  |   6 +-
 .../api/tests/test_certificate_use_case_4.py  |   8 +-
 .../api/tests/test_certificate_use_case_5.py  |   8 +-
 rootfs/api/tests/test_config.py               |  34 ++--
 rootfs/api/tests/test_domain.py               |  32 +--
 rootfs/api/tests/test_event.py                |   2 +-
 rootfs/api/tests/test_gateway.py              |  38 ++--
 rootfs/api/tests/test_procfile.py             |  32 +--
 rootfs/api/tests/test_ptypes.py               |   4 +-
 rootfs/api/tests/test_release.py              |   8 +-
 rootfs/api/tests/test_service.py              |  22 +--
 rootfs/api/tests/test_tls.py                  |   4 +-
 rootfs/api/urls.py                            |  10 +-
 rootfs/api/utils.py                           |  38 ++--
 rootfs/api/views.py                           |  97 +++++----
 rootfs/scheduler/resources/service.py         |   4 +-
 rootfs/scheduler/tests/test_gateway.py        |  12 +-
 37 files changed, 488 insertions(+), 457 deletions(-)
 create mode 100644 rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
 rename rootfs/api/migrations/{0012_alter_app_options_certificate_app_release_conditions_and_more.py => 0013_migration_permissions_and_certificates.py} (53%)

diff --git a/rootfs/api/admissions.py b/rootfs/api/admissions.py
index b7c1e5a75..48b050faf 100644
--- a/rootfs/api/admissions.py
+++ b/rootfs/api/admissions.py
@@ -28,8 +28,8 @@ def detect(self, request: Request) -> bool:
     def handle(self, request: Request) -> bool:
         app_id = request["object"]["metadata"]["namespace"]
         app = models.app.App.objects.filter(id=app_id).first()
-        container_type = request["object"]["metadata"].get("labels", {}).get("type", "")
-        if app and container_type:
+        ptype = request["object"]["metadata"].get("labels", {}).get("type", "")
+        if app and ptype:
             status = request["object"]["status"]
             replicas = request["object"]["spec"].get("replicas", 0)
             if "active" in status:
@@ -37,11 +37,11 @@ def handle(self, request: Request) -> bool:
             elif "succeeded" in status or "failed" in status:
                 replicas -= 1
             replicas = 0 if replicas < 0 else replicas
-            if app.structure.get(container_type, 0) != replicas:
+            if app.structure.get(ptype, 0) != replicas:
                 models.app.App.objects.filter(id=app.id).update(
                     structure=Func(
                         F("structure"),
-                        Value([container_type]),
+                        Value([ptype]),
                         Value(replicas, JSONField()),
                         function="jsonb_set",
                     )
@@ -64,18 +64,18 @@ def detect(self, request: Request) -> bool:
     def handle(self, request: Request) -> bool:
         app_id = request["object"]["metadata"]["namespace"]
         app = models.app.App.objects.filter(id=app_id).first()
-        container_type = None
+        ptype = None
         for item in request["object"]["status"]["selector"].split(","):
             key, value = item.split("=")
             if key == "type":
-                container_type = value
-        if app and container_type:
+                ptype = value
+        if app and ptype:
             replicas = request["object"]["spec"].get("replicas", 0)
-            if app.structure.get(container_type, 0) != replicas:
+            if app.structure.get(ptype, 0) != replicas:
                 models.app.App.objects.filter(id=app.id).update(
                     structure=Func(
                         F("structure"),
-                        Value([container_type]),
+                        Value([ptype]),
                         Value(replicas, JSONField()),
                         function="jsonb_set",
                     )
diff --git a/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py b/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
new file mode 100644
index 000000000..730d3bd36
--- /dev/null
+++ b/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
@@ -0,0 +1,58 @@
+# Generated by Django 4.2.15 on 2024-09-03 03:48
+
+import api.utils
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_appsettings_autodeploy'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='app',
+            options={'ordering': ['id'], 'verbose_name': 'Application'},
+        ),
+        migrations.RenameField(
+            model_name='domain',
+            old_name='procfile_type',
+            new_name='ptype',
+        ),
+        migrations.RenameField(
+            model_name='route',
+            old_name='procfile_type',
+            new_name='ptype',
+        ),
+        migrations.RenameField(
+            model_name='service',
+            old_name='procfile_type',
+            new_name='ptype',
+        ),
+        migrations.AddField(
+            model_name='certificate',
+            name='app',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='api.app'),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='conditions',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name='certificate',
+            name='name',
+            field=models.CharField(max_length=253, validators=[api.utils.validate_label]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='certificate',
+            unique_together={('app', 'name')},
+        ),
+        migrations.AlterUniqueTogether(
+            name='service',
+            unique_together={('app', 'ptype')},
+        ),
+    ]
diff --git a/rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
similarity index 53%
rename from rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py
rename to rootfs/api/migrations/0013_migration_permissions_and_certificates.py
index fcda371e5..134574746 100644
--- a/rootfs/api/migrations/0012_alter_app_options_certificate_app_release_conditions_and_more.py
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -1,8 +1,6 @@
-# Generated by Django 4.2.15 on 2024-08-30 00:53
+# Generated by Django 4.2.15 on 2024-09-03 03:48
 
-import api.utils
-from django.db import migrations, models
-import django.db.models.deletion
+from django.db import migrations
 from guardian.shortcuts import assign_perm, get_users_with_perms, remove_perm
 from api.models.app import App, VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
 from api.models.domain import Domain
@@ -34,34 +32,10 @@ def migration_certificate(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0011_appsettings_autodeploy'),
+        ('api', '0012_alter_app_options_rename_procfile_type_domain_ptype_and_more'),
     ]
 
     operations = [
-        migrations.AlterModelOptions(
-            name='app',
-            options={'ordering': ['id'], 'verbose_name': 'Application'},
-        ),
-        migrations.AddField(
-            model_name='certificate',
-            name='app',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='api.app'),
-            preserve_default=False,
-        ),
-        migrations.AddField(
-            model_name='release',
-            name='conditions',
-            field=models.JSONField(default=list),
-        ),
-        migrations.AlterField(
-            model_name='certificate',
-            name='name',
-            field=models.CharField(max_length=253, validators=[api.utils.validate_label]),
-        ),
-        migrations.AlterUniqueTogether(
-            name='certificate',
-            unique_together={('app', 'name')},
-        ),
         migrations.RunPython(migration_permission),
         migrations.RunPython(migration_certificate),
     ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index baa4a04cc..305d1f595 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -31,7 +31,7 @@
 from .tls import TLS
 from .appsettings import AppSettings
 from .volume import Volume
-from .base import UuidAuditedModel, PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN, DEFAULT_HTTP_PORT
+from .base import UuidAuditedModel, PTYPE_WEB, PTYPE_RUN, DEFAULT_HTTP_PORT
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -148,21 +148,21 @@ def save(self, *args, **kwargs):
         return application
 
     @property
-    def procfile_types(self):
+    def ptypes(self):
         return list(self.structure.keys())
 
-    def check_procfile_types(self, procfile_types):
+    def check_ptypes(self, ptypes):
         """
         check available procfile types
         """
-        if not procfile_types:
-            procfile_types = self.procfile_types
+        if not ptypes:
+            ptypes = self.ptypes
         else:
-            invalid_procfile_types = procfile_types.difference(self.procfile_types)
-            if len(invalid_procfile_types) != 0:
+            invalid_ptypes = ptypes.difference(self.ptypes)
+            if len(invalid_ptypes) != 0:
                 raise DryccException("process type {} is not included in procfile".
-                                     format(','.join(invalid_procfile_types)))
-        return procfile_types
+                                     format(','.join(invalid_ptypes)))
+        return ptypes
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
@@ -261,8 +261,8 @@ def restart(self, **kwargs):  # noqa
     def scale(self, user, structure):
         err_msg = None
         release = self.release_set.filter(failed=False).latest()
-        if (PROCFILE_TYPE_RUN in structure or release.build is None):
-            if PROCFILE_TYPE_RUN in structure:
+        if (PTYPE_RUN in structure or release.build is None):
+            if PTYPE_RUN in structure:
                 err_msg = 'Cannot set scale for reserved types, procfile type is: run'
             else:
                 err_msg = 'No build associated with this release'
@@ -271,12 +271,12 @@ def scale(self, user, structure):
         app_settings = self.appsettings_set.latest()
         self._scale(user, structure, release, app_settings)
 
-    def pipeline(self, release, procfile_types=None, force_deploy=False):
+    def pipeline(self, release, ptypes=None, force_deploy=False):
         prefix = f"[pipeline] release {release.version_name}"
         try:
             if release.build is not None:
                 if release.build.dryccfile:
-                    for run in release.get_runners(procfile_types):
+                    for run in release.get_runners(ptypes):
                         self.log(f"{prefix} starts running pipeline.run {run['image']}")
                         job_name = self.run(
                             self.owner, run['image'], command=run['command'],
@@ -292,21 +292,21 @@ def pipeline(self, release, procfile_types=None, force_deploy=False):
                 rollback_on_failure = self.appsettings_set.latest().autorollback
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
-                self.deploy(release, procfile_types, force_deploy, rollback_on_failure)
+                self.deploy(release, ptypes, force_deploy, rollback_on_failure)
             if release.state == "created":
                 release.state = "succeed"
-            release.add_condition(state="succeed", action="pipeline", ptypes=procfile_types)
+            release.add_condition(state="succeed", action="pipeline", ptypes=ptypes)
         except Exception as e:
             release.failed, release.state = True, "crashed"
             release.add_condition(
-                state="crashed", action="pipeline", ptypes=procfile_types, exception=str(e))
+                state="crashed", action="pipeline", ptypes=ptypes, exception=str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
         finally:
-            DeployLock(self.pk).release(procfile_types)
+            DeployLock(self.pk).release(ptypes)
             release.save(update_fields=["state", "failed"])  # avoid overwriting other fields
         self.log(f"{prefix} run completed...")
 
-    def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_failure=True):
+    def deploy(self, release, ptypes=None, force_deploy=False, rollback_on_failure=True):
         """
         Deploy a new release to this application
 
@@ -324,8 +324,8 @@ def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_f
             prev_release and prev_release.build and prev_release.build.type != release.build.type
         ):
             # structure {} or build type change, merge old structure if exists
-            for procfile_type, scale in default_structure.items():
-                default_structure[procfile_type] = self.structure.get(procfile_type, scale)
+            for ptype, scale in default_structure.items():
+                default_structure[ptype] = self.structure.get(ptype, scale)
             self.structure = default_structure
             self.save()
         # deploy application to k8s. Also handles initial scaling
@@ -333,16 +333,16 @@ def deploy(self, release, procfile_types=None, force_deploy=False, rollback_on_f
         volumes = self.volume_set.all()
         deploys = {}
         for scale_type, replicas in self.structure.items():
-            if procfile_types is not None and scale_type not in procfile_types:
+            if ptypes is not None and scale_type not in ptypes:
                 continue
             scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
             deploys[scale_type] = self._gather_app_settings(
                 release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
         self._deploy(
-            deploys, procfile_types, prev_release, release, force_deploy, rollback_on_failure)
+            deploys, ptypes, prev_release, release, force_deploy, rollback_on_failure)
         # cleanup old release objects from kubernetes
-        self.cleanup_old(procfile_types)
-        release.cleanup_old(procfile_types)
+        self.cleanup_old(ptypes)
+        release.cleanup_old(ptypes)
 
     def mount(self, user, volume, structure=None):
         if self.release_set.filter(failed=False).latest().build is None:
@@ -351,13 +351,13 @@ def mount(self, user, volume, structure=None):
         app_settings = self.appsettings_set.latest()
         self._mount(user, volume, release, app_settings, structure=structure)
 
-    def cleanup_old(self, procfile_types=None):
+    def cleanup_old(self, ptypes=None):
         names = []
         for scale_type in self.structure.keys():
             names.append(self._get_job_id(scale_type))
         labels = {'heritage': 'drycc'}
-        if procfile_types is not None:
-            labels["type__in"] = procfile_types
+        if ptypes is not None:
+            labels["type__in"] = ptypes
         deployments = self.scheduler().deployments.get(self.id, labels=labels).json()["items"]
         if deployments is not None:
             for deployment in deployments:
@@ -380,27 +380,27 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         volume_list = []
         if volumes:
             for volume in Volume.objects.filter(app=self, name__in=volumes.keys()):
-                volume.path[PROCFILE_TYPE_RUN] = volumes.get(volume.name, None)
+                volume.path[PTYPE_RUN] = volumes.get(volume.name, None)
                 volume_list.append(volume)
         else:
             for volume in Volume.objects.filter(app=self):
-                if PROCFILE_TYPE_RUN in volume.path.keys():
+                if PTYPE_RUN in volume.path.keys():
                     volume_list.append(volume)
         data = self._gather_app_settings(
-            release, app_settings, procfile_type=PROCFILE_TYPE_RUN,
+            release, app_settings, ptype=PTYPE_RUN,
             replicas=1, volumes=volume_list)
         data['restart_policy'] = 'Never'
         data['active_deadline_seconds'] = timeout
         data['ttl_seconds_after_finished'] = expires
-        name = self._get_job_id(PROCFILE_TYPE_RUN) + '-' + pod_name()
+        name = self._get_job_id(PTYPE_RUN) + '-' + pod_name()
         self.log("{} on {} runs '{}'".format(user.username, name, command))
         kwargs.update(data)
         try:
             # create application config and build the pod manifest
-            self.set_application_config(release, PROCFILE_TYPE_RUN)
+            self.set_application_config(release, PTYPE_RUN)
             self.scheduler().job.create(self.id, name, image, command, args, **kwargs)
         except Exception as e:
-            err = '{} ({}): {}'.format(name, PROCFILE_TYPE_RUN, e)
+            err = '{} ({}): {}'.format(name, PTYPE_RUN, e)
             raise ServiceUnavailable(err) from e
         return name
 
@@ -582,7 +582,7 @@ def autoscale(self, proc_type, autoscale):
         """
         Set autoscale rules for the application
         """
-        if proc_type == PROCFILE_TYPE_RUN:
+        if proc_type == PTYPE_RUN:
             raise DryccException('Cannot set autoscale for reserved types, procfile type is: run')
         name = '{}-{}'.format(self.id, proc_type)
         # basically fake out a Deployment object (only thing we use) to assign to the HPA
@@ -643,23 +643,23 @@ def state_to_k8s(self):
         if release.build is None:
             self.log('the last release does not have a build, skipping deployment...')
             return
-        procfile_types = set()
-        for procfile_type, scale in self.structure.items():
+        ptypes = set()
+        for ptype, scale in self.structure.items():
             response = self.scheduler().deployment.get(
-                self.id, self._get_job_id(procfile_type),
+                self.id, self._get_job_id(ptype),
                 ignore_exception=True)
             if response.status_code == 404 and scale > 0:
-                procfile_types.add(procfile_type)
+                ptypes.add(ptype)
             elif response.status_code != 200:
                 data = response.json()
                 self.log('get deployment status_code {}, message: {}'.format(
                     response.status_code, data.get("message", "")), logging.ERROR)
-        if len(procfile_types) == 0:
+        if len(ptypes) == 0:
             self.log('the cluster status is the latest, skipping deployment...')
             return
-        self.deploy(release, procfile_types, False, False)
+        self.deploy(release, ptypes, False, False)
 
-    def set_application_config(self, release, procfile_type):
+    def set_application_config(self, release, ptype):
         """
         Creates the application config as a secret in Kubernetes and
         updates it if it already exists
@@ -667,19 +667,19 @@ def set_application_config(self, release, procfile_type):
         # env vars are stored in secrets and mapped to env in k8s
         labels = {
             'version': release.version_name,
-            'type': procfile_type,
+            'type': ptype,
             'class': 'env'
         }
 
         # secrets use dns labels for keys, map those properly here
         secrets_env = {}
-        for key, value in self._build_env_vars(release, procfile_type).items():
+        for key, value in self._build_env_vars(release, ptype).items():
             secrets_env[key.lower().replace('_', '-')] = str(value)
 
         # dictionary sorted by key
         secrets_env = OrderedDict(sorted(secrets_env.items(), key=lambda t: t[0]))
 
-        secret_name = "{}-{}-{}-env".format(self.id, procfile_type, release.version_name)
+        secret_name = "{}-{}-{}-env".format(self.id, ptype, release.version_name)
         try:
             self.scheduler().secret.get(self.id, secret_name)
         except KubeHTTPException:
@@ -690,8 +690,8 @@ def set_application_config(self, release, procfile_type):
     def to_measurements(self, timestamp: float):
         measurements = []
         config = self.config_set.latest()
-        for container_type, scale in self.structure.items():
-            plan = config.limits.get(container_type)
+        for ptype, scale in self.structure.items():
+            plan = config.limits.get(ptype)
             measurements.append({
                 "app_id": str(self.uuid),
                 "owner": self.owner_id,
@@ -700,7 +700,7 @@ def to_measurements(self, timestamp: float):
                 "unit": "number",
                 "usage": scale,
                 "kwargs": {
-                    "procfile_type": container_type,
+                    "ptype": ptype,
                 },
                 "timestamp": int(timestamp),
             })
@@ -709,8 +709,8 @@ def to_measurements(self, timestamp: float):
     def __str__(self):
         return self.id
 
-    def _get_job_id(self, container_type):
-        return f"{self.id}-{container_type}"
+    def _get_job_id(self, ptype):
+        return f"{self.id}-{ptype}"
 
     def _clean_app_logs(self):
         """Delete application logs stored by the logger component"""
@@ -728,7 +728,7 @@ def _mount(self, user, volume, release, app_settings, structure=None):
         volumes = Volume.objects.filter(app=self)
         tasks = []
         for scale_type, replicas in structure.items() if structure else self.structure.items():
-            if scale_type != PROCFILE_TYPE_RUN:
+            if scale_type != PTYPE_RUN:
                 replicas = self.structure.get(scale_type, 0)
                 scale_type_volumes = [
                     volume for volume in volumes if scale_type in volume.path.keys()]
@@ -759,7 +759,7 @@ def _mount(self, user, volume, release, app_settings, structure=None):
             raise ServiceUnavailable(err) from e
         self.log(f'{user.username} changed volume mount for {volume}')
 
-    def _deploy(self, deploys, procfile_types, prev_release,
+    def _deploy(self, deploys, ptypes, prev_release,
                 release, force_deploy, rollback_on_failure):
         # Sort deploys so routable comes first
         deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
@@ -790,7 +790,7 @@ def _deploy(self, deploys, procfile_types, prev_release,
                     # This goes in the log before the rollback starts
                     self.log(err, logging.ERROR)
                     # revert all process types to old release
-                    self.deploy(prev_release, procfile_types, True, False)
+                    self.deploy(prev_release, ptypes, True, False)
                     # let it bubble up
                     raise DryccException('{}\n{}'.format(err, str(e))) from e
 
@@ -801,19 +801,19 @@ def _deploy(self, deploys, procfile_types, prev_release,
             err = '(app::deploy): {}'.format(e)
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
-        for procfile_type in deploys.keys():
-            if procfile_type == PROCFILE_TYPE_WEB:  # http
-                target_port = release.get_port(procfile_type)
+        for ptype in deploys.keys():
+            if ptype == PTYPE_WEB:  # http
+                target_port = release.get_port(ptype)
                 self._create_default_ingress(target_port)
-            service = self.service_set.filter(procfile_type=procfile_type).first()
+            service = self.service_set.filter(ptype=ptype).first()
             if not service:
                 continue
             if prev_release and prev_release.build:
                 continue
-            if procfile_type == PROCFILE_TYPE_WEB:
-                self._verify_http_health(service, **deploys[procfile_type])
+            if ptype == PTYPE_WEB:
+                self._verify_http_health(service, **deploys[ptype])
             else:
-                self._verify_tcp_health(service, **deploys[procfile_type])
+                self._verify_tcp_health(service, **deploys[ptype])
 
     def _scale(self, user, structure, release, app_settings):  # noqa
         """Scale containers up or down to match requested structure."""
@@ -828,11 +828,11 @@ def _scale(self, user, structure, release, app_settings):  # noqa
             raise DryccException('Invalid scaling format: {}'.format(e))
 
         # test for available process types
-        for container_type in structure:
-            if container_type not in (PROCFILE_TYPE_WEB, PROCFILE_TYPE_RUN) and \
-                    container_type not in release.procfile_types:
+        for ptype in structure:
+            if ptype not in (PTYPE_WEB, PTYPE_RUN) and \
+                    ptype not in release.ptypes:
                 raise NotFound(
-                    'Container type {} does not exist in application'.format(container_type))
+                    'Container type {} does not exist in application'.format(ptype))
 
         # merge current structure and the new items together
         old_structure = self.structure
@@ -883,24 +883,24 @@ def _scale_pods(self, scale_types, release, app_settings):
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
 
-    def _set_default_limit(self, config, procfile_type):
-        if procfile_type not in config.limits:
+    def _set_default_limit(self, config, ptype):
+        if ptype not in config.limits:
             plan = LimitPlan.get_default()
-            config.limits[procfile_type] = plan.id
+            config.limits[ptype] = plan.id
             config.save(update_fields=['limits'])
         return config
 
     def _set_default_config(self):
         plan = LimitPlan.get_default()
-        limits = {PROCFILE_TYPE_WEB: plan.id, PROCFILE_TYPE_RUN: plan.id}
+        limits = {PTYPE_WEB: plan.id, PTYPE_RUN: plan.id}
         try:
             config = self.config_set.latest()
-            limits[PROCFILE_TYPE_WEB] = config.limits.get(PROCFILE_TYPE_WEB, plan.id)
-            limits[PROCFILE_TYPE_RUN] = config.limits.get(PROCFILE_TYPE_RUN, plan.id)
+            limits[PTYPE_WEB] = config.limits.get(PTYPE_WEB, plan.id)
+            limits[PTYPE_RUN] = config.limits.get(PTYPE_RUN, plan.id)
         except Config.DoesNotExist:
             config = Config.objects.create(owner=self.owner, app=self, limits=limits)
-        for procfile_type in self.procfile_types:
-            limits[procfile_type] = config.limits.get(procfile_type, plan.id)
+        for ptype in self.ptypes:
+            limits[ptype] = config.limits.get(ptype, plan.id)
         if limits != config.limits:
             config.limits = limits
             config.save(update_fields=['limits'])
@@ -909,9 +909,9 @@ def _set_default_config(self):
     def _create_default_ingress(self, target_port):
         # create default service
         try:
-            service = self.service_set.filter(procfile_type=PROCFILE_TYPE_WEB).latest()
+            service = self.service_set.filter(ptype=PTYPE_WEB).latest()
         except Service.DoesNotExist:
-            service = Service(owner=self.owner, app=self, procfile_type=PROCFILE_TYPE_WEB)
+            service = Service(owner=self.owner, app=self, ptype=PTYPE_WEB)
             service.add_port(DEFAULT_HTTP_PORT, "TCP", target_port)
             service.save()
         else:
@@ -935,7 +935,7 @@ def _create_default_ingress(self, target_port):
                 route.save()
         except Route.DoesNotExist:
             route = Route(app=self, owner=self.owner, kind="HTTPRoute", name=self.id,
-                          port=DEFAULT_HTTP_PORT, procfile_type=service.procfile_type)
+                          port=DEFAULT_HTTP_PORT, ptype=service.ptype)
             route.rules = route.default_rules
             attached, msg = route.attach(gateway.name, DEFAULT_HTTP_PORT)
             if not attached:
@@ -1035,10 +1035,10 @@ def _check_deployment_in_progress(self, deploys, force_deploy=False):
     @staticmethod
     def _default_structure(release):
         """Scale to default structure based on release type"""
-        structure = {PROCFILE_TYPE_WEB: 1}
-        for procfile_type in release.procfile_types:
-            if procfile_type != PROCFILE_TYPE_WEB:
-                structure[procfile_type] = 0
+        structure = {PTYPE_WEB: 1}
+        for ptype in release.ptypes:
+            if ptype != PTYPE_WEB:
+                structure[ptype] = 0
         return structure
 
     def _scheduler_filter(self, **kwargs):
@@ -1053,7 +1053,7 @@ def _scheduler_filter(self, **kwargs):
             labels.update({'version': version})
         return labels
 
-    def _build_env_vars(self, release, procfile_type):
+    def _build_env_vars(self, release, ptype):
         """
         Build a dict of env vars, setting default vars based on app type
         and then combining with the user set ones
@@ -1073,9 +1073,9 @@ def _build_env_vars(self, release, procfile_type):
         default_env['SOURCE_VERSION'] = release.build.sha
         # merge envs on top of default to make envs win
         default_env.update(release.config.values)
-        default_env.update(release.config.typed_values.get(procfile_type, {}))
+        default_env.update(release.config.typed_values.get(ptype, {}))
         # fetch application port and inject into ENV vars as needed
-        port = release.get_port(procfile_type)
+        port = release.get_port(ptype)
         if port:
             default_env['PORT'] = port
         return default_env
@@ -1117,7 +1117,7 @@ def _get_private_registry_config(self, image, registry=None):
         })
         return docker_config, name, True
 
-    def _get_volumes_and_mounts(self, procfile_type, volumes):
+    def _get_volumes_and_mounts(self, ptype, volumes):
         k8s_volumes, k8s_volume_mounts = [], []
         if volumes:
             for volume in volumes:
@@ -1128,10 +1128,10 @@ def _get_volumes_and_mounts(self, procfile_type, volumes):
                     k8s_volume.update(volume.parameters)
                 k8s_volumes.append(k8s_volume)
                 k8s_volume_mounts.append(
-                    {"name": volume.name, "mountPath": volume.path.get(procfile_type)})
+                    {"name": volume.name, "mountPath": volume.path.get(ptype)})
         return k8s_volumes, k8s_volume_mounts
 
-    def _gather_app_settings(self, release, app_settings, procfile_type, replicas, volumes=None):
+    def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=None):
         """
         Gathers all required information needed in one easy place for passing into
         the Kubernetes client to deploy an application
@@ -1139,10 +1139,10 @@ def _gather_app_settings(self, release, app_settings, procfile_type, replicas, v
         Any global setting that can also be set per app goes here
         """
 
-        envs = self._build_env_vars(release, procfile_type)
+        envs = self._build_env_vars(release, ptype)
         # Obtain a limit plan that must exist, if raise error here, it must be a bug
-        config = self._set_default_limit(release.config, procfile_type)
-        limit_plan = LimitPlan.objects.get(id=config.limits.get(procfile_type))
+        config = self._set_default_limit(release.config, ptype)
+        limit_plan = LimitPlan.objects.get(id=config.limits.get(ptype))
 
         # see if the app config has deploy batch preference, otherwise use global
         batches = int(config.values.get('DRYCC_DEPLOY_BATCHES', settings.DRYCC_DEPLOY_BATCHES))
@@ -1166,22 +1166,22 @@ def _gather_app_settings(self, release, app_settings, procfile_type, replicas, v
 
         # create image pull secret if needed
         image_pull_secret_name = self.image_pull_secret(
-            self.id, config.registry, release.get_deploy_image(procfile_type))
+            self.id, config.registry, release.get_deploy_image(ptype))
 
         # only web is routable
         # https://www.drycc.cc/applications/managing-app-processes/#default-process-types
         routable = True if (
-            procfile_type == PROCFILE_TYPE_WEB and app_settings.routable) else False
+            ptype == PTYPE_WEB and app_settings.routable) else False
 
-        healthcheck = config.healthcheck.get(procfile_type, {})
-        volumes, volume_mounts = self._get_volumes_and_mounts(procfile_type, volumes)
+        healthcheck = config.healthcheck.get(ptype, {})
+        volumes, volume_mounts = self._get_volumes_and_mounts(ptype, volumes)
         return {
-            'tags': config.tags.get(procfile_type, {}),
+            'tags': config.tags.get(ptype, {}),
             'envs': envs,
             'registry': config.registry,
             'replicas': replicas,
             'version': release.version_name,
-            'app_type': procfile_type,
+            'app_type': ptype,
             'resources': {"limits": limit_plan.limits, "requests": limit_plan.requests},
             'build_type': release.build.type,
             'annotations': limit_plan.annotations,
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 87ae5353a..7e66d6136 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -27,12 +27,12 @@
         "access_token", "expires_in", "token_type", "scope", "refresh_token"
     ],
 }
-PROCFILE_TYPE_WEB = "web"
-PROCFILE_TYPE_RUN = "run"
+PTYPE_WEB = "web"
+PTYPE_RUN = "run"
 DEFAULT_HTTP_PORT = 80
 DEFAULT_HTTPS_PORT = 443
-PROCFILE_TYPE_MIN_LENGTH = 3
-PROCFILE_TYPE_MAX_LENGTH = 63
+PTYPE_MIN_LENGTH = 3
+PTYPE_MAX_LENGTH = 63
 
 
 def get_anonymous_user_instance(user): return user(id=-1, username=settings.ANONYMOUS_USER_NAME)
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 331b3cf9c..4d0f300b6 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -42,7 +42,7 @@ def type(self):
             return 'image'
 
     @property
-    def procfile_types(self):
+    def ptypes(self):
         if self.dryccfile:
             return list(self.dryccfile['deploy'].keys())
         return list(self.procfile.keys())
@@ -60,13 +60,13 @@ def source_based(self):
     def version(self):
         return 'git-{}'.format(self.sha) if self.source_based else 'latest'
 
-    def get_image(self, procfile_type, default_image=None):
+    def get_image(self, ptype, default_image=None):
         docker = self.dryccfile.get('build', {}).get('docker', {})
-        if procfile_type in docker:
-            if procfile_type == 'web':
+        if ptype in docker:
+            if ptype == 'web':
                 return self.image
             else:
-                return f'{self.image}-{procfile_type}'
+                return f'{self.image}-{ptype}'
         return default_image if default_image else self.image
 
     def create_release(self, user, *args, **kwargs):
@@ -108,8 +108,8 @@ def save(self, **kwargs):
             # previous release had a Procfile and the current one does not
             (
                 previous_release.build is not None and
-                len(previous_release.procfile_types) > 0 and
-                len(self.procfile_types) == 0
+                len(previous_release.ptypes) > 0 and
+                len(self.ptypes) == 0
             )
         ):
             # Reject deployment
@@ -125,16 +125,16 @@ def save(self, **kwargs):
             # previous release had a Procfile and the current one does as well
             (
                 previous_release.build is not None and
-                len(previous_release.procfile_types) > 0 and
-                len(self.procfile_types) > 0
+                len(previous_release.ptypes) > 0 and
+                len(self.ptypes) > 0
             )
         )
 
         # spin down any proc type removed between the last procfile and the newest one
         if remove_procs and previous_release.build is not None:
             removed = {}
-            for proc in previous_release.procfile_types:
-                if proc not in self.procfile_types and self.app.structure.get(proc, 0) > 0:
+            for proc in previous_release.ptypes:
+                if proc not in self.ptypes and self.app.structure.get(proc, 0) > 0:
                     # Scale proc type down to 0
                     removed[proc] = 0
 
@@ -145,8 +145,8 @@ def save(self, **kwargs):
         if (
             settings.DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE is False and
             previous_release.build is not None and
-            len(previous_release.procfile_types) > 0 and
-            len(self.procfile_types) == 0
+            len(previous_release.ptypes) > 0 and
+            len(self.ptypes) == 0
         ):
             self.procfile = previous_release.build.procfile
             self.dryccfile = previous_release.build.dryccfile
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 892d11454..e28633ef2 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -15,9 +15,9 @@ class Config(UuidAuditedModel):
     Set of configuration values applied as environment variables
     during runtime execution of the Application.
     """
-    procfile_fields = ("lifecycle_post_start", "lifecycle_pre_stop", "tags", "limits",
-                       "typed_values", "healthcheck", "termination_grace_period")
-    all_diff_fields = ("values", "registry") + procfile_fields
+    ptype_fields = ("lifecycle_post_start", "lifecycle_pre_stop", "tags", "limits",
+                    "typed_values", "healthcheck", "termination_grace_period")
+    allof_fields = ("values", "registry") + ptype_fields
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
@@ -59,7 +59,7 @@ def previous(self):
     def diff(self, config=None):
         old_config = config if config else self.previous()
         result = {}
-        for field in self.all_diff_fields:
+        for field in self.allof_fields:
             result[field] = dict_diff(getattr(self, field), getattr(old_config, field))
         return result
 
@@ -118,27 +118,27 @@ def _set_tags(self, previous_config):
         data = getattr(previous_config, 'tags', {}).copy()
         new_data = getattr(self, 'tags', {}).copy()
         # remove config keys if a null value is provided
-        for procfile_type, values in new_data.items():
+        for ptype, values in new_data.items():
             if not values:
                 # error if unsetting non-existing key
-                if procfile_type not in data:
+                if ptype not in data:
                     raise UnprocessableEntity(
-                        '{} does not exist under {}'.format(procfile_type, 'tags'))
-                data.pop(procfile_type)
+                        '{} does not exist under {}'.format(ptype, 'tags'))
+                data.pop(ptype)
             else:
                 if not self.scheduler().node.get(labels=values).json()['items']:
                     labels = ['{}={}'.format(key, value) for key, value in values.items()]
                     message = 'No nodes matched the provided labels: {}'.format(', '.join(labels))
                     # Find out if there are any other tags around
-                    old_tags = previous_config.tags.get(procfile_type, {})
+                    old_tags = previous_config.tags.get(ptype, {})
                     if old_tags:
                         old = ['{}={}'.format(key, value) for key, value in old_tags.items()]
                         new = set(labels) - set(old)
                         if new:
                             message += ' - Addition of {} is the cause'.format(', '.join(new))
                     raise DryccException(message)
-                data[procfile_type] = self._merge_data(
-                    'tags', data.get(procfile_type, {}), values)
+                data[ptype] = self._merge_data(
+                    'tags', data.get(ptype, {}), values)
         setattr(self, 'tags', data)
 
     def _set_limits(self, previous_config):
@@ -147,7 +147,7 @@ def _set_limits(self, previous_config):
         # check procfile
         for key, value in new_data.items():
             if value is None:
-                if key in self.app.procfile_types:
+                if key in self.app.ptypes:
                     raise UnprocessableEntity(
                         "the %s has already been used and cannot be deleted" % key)
         self._merge_data('limits', data, new_data)
@@ -172,14 +172,14 @@ def _set_typed_values(self, previous_config):
         data = getattr(previous_config, 'typed_values', {}).copy()
         new_data = getattr(self, 'typed_values', {}).copy()
         # remove config keys if a null value is provided
-        for procfile_type, values in new_data.items():
+        for ptype, values in new_data.items():
             if not values:
                 # error if unsetting non-existing key
-                if procfile_type not in data:
+                if ptype not in data:
                     raise UnprocessableEntity(
-                        '{} does not exist under {}'.format(procfile_type, 'typed_values'))
-                data.pop(procfile_type)
+                        '{} does not exist under {}'.format(ptype, 'typed_values'))
+                data.pop(ptype)
             else:
-                data[procfile_type] = self._merge_data(
-                    'typed_values', data.get(procfile_type, {}), values)
+                data[ptype] = self._merge_data(
+                    'typed_values', data.get(ptype, {}), values)
         setattr(self, 'typed_values', data)
diff --git a/rootfs/api/models/domain.py b/rootfs/api/models/domain.py
index a77952905..bb5e8db71 100644
--- a/rootfs/api/models/domain.py
+++ b/rootfs/api/models/domain.py
@@ -1,7 +1,7 @@
 from django.db import models
 from django.db import transaction
 from django.contrib.auth import get_user_model
-from .base import AuditedModel, PROCFILE_TYPE_MAX_LENGTH
+from .base import AuditedModel, PTYPE_MAX_LENGTH
 
 User = get_user_model()
 
@@ -21,7 +21,7 @@ class Domain(AuditedModel):
         blank=True,
         null=True
     )
-    procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
+    ptype = models.CharField(max_length=PTYPE_MAX_LENGTH)
 
     class Meta:
         ordering = ['domain', 'certificate']
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index c5cc5fa5d..a411309c4 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -7,7 +7,7 @@
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 
-from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, PROCFILE_TYPE_MAX_LENGTH
+from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, PTYPE_MAX_LENGTH
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -183,7 +183,7 @@ class Route(AuditedModel):
     rules = models.JSONField(default=list)
     routable = models.BooleanField(default=True)
     parent_refs = models.JSONField(default=list)
-    procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
+    ptype = models.CharField(max_length=PTYPE_MAX_LENGTH)
 
     @property
     def protocols(self):
@@ -194,11 +194,11 @@ def protocols(self):
     @property
     def hostnames(self):
         return [domain.domain for domain in self.app.domain_set.filter(
-                procfile_type=self.procfile_type)]
+                ptype=self.ptype)]
 
     @property
     def default_rules(self):
-        service = get_object_or_404(self.app.service_set, procfile_type=self.procfile_type)
+        service = get_object_or_404(self.app.service_set, ptype=self.ptype)
         backend_refs = []
         for item in service.ports:
             if item["port"] == self.port:
@@ -223,7 +223,7 @@ def log(self, message, level=logging.INFO):
 
     def check_rules(self):
         service = self.app.service_set.filter(
-            procfile_type=self.procfile_type).first()
+            ptype=self.ptype).first()
         ports = [item["port"] for item in service.ports]
         for rule in self.rules:
             for backend_ref in rule["backendRefs"]:
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index ec4785888..7d4febd15 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -54,23 +54,23 @@ def __str__(self):
         return "{0}-{1}".format(self.app.id, self.version_name)
 
     @property
-    def procfile_types(self):
+    def ptypes(self):
         if self.build is not None:
-            return self.build.procfile_types
+            return self.build.ptypes
         return []
 
     @property
     def version_name(self):
         return f'v{self.version}'
 
-    def get_runners(self, procfile_types):
+    def get_runners(self, ptypes):
         results = []
-        procfile_types = self.procfile_types if not procfile_types else procfile_types
+        ptypes = self.ptypes if not ptypes else ptypes
         for run in self.build.dryccfile.get('run', []):
-            for container_type in procfile_types:
+            for ptype in ptypes:
                 when_ptypes = run.get('when', {}).get('ptypes', [])
-                if not when_ptypes or container_type in when_ptypes:
-                    image = run.get('image', self.build.get_image(container_type))
+                if not when_ptypes or ptype in when_ptypes:
+                    image = run.get('image', self.build.get_image(ptype))
                     results.append({
                         'image': self.build.get_image(image, default_image=image),
                         'args': run.get('args', []),
@@ -92,39 +92,39 @@ def add_condition(self, **kwargs):
             )
         )
 
-    def get_deploy_image(self, container_type):
+    def get_deploy_image(self, ptype):
         """
         In the deploy phase of dryccfile
         Return the kubernetes "container image" to be sent off to the scheduler.
         """
-        image = self.build.dryccfile.get('deploy', {}).get(container_type, {}).get(
-            'image', self.build.get_image(container_type))
+        image = self.build.dryccfile.get('deploy', {}).get(ptype, {}).get(
+            'image', self.build.get_image(ptype))
         return self.build.get_image(image, default_image=image)
 
-    def get_deploy_args(self, container_type):
+    def get_deploy_args(self, ptype):
         """
         In the deploy phase of dryccfile
         Return the kubernetes "container arguments" to be sent off to the scheduler.
         """
         if self.build is not None:
             if self.build.dryccfile:
-                return self.build.dryccfile['deploy'].get(container_type, {}).get('args', [])
+                return self.build.dryccfile['deploy'].get(ptype, {}).get('args', [])
             else:
                 # dockerfile or container image
                 if self.build.dockerfile or not self.build.sha:
                     # has profile
-                    if self.build.procfile and container_type in self.build.procfile:
-                        args = self.build.procfile[container_type]
+                    if self.build.procfile and ptype in self.build.procfile:
+                        args = self.build.procfile[ptype]
                         return args.split()
         return []
 
-    def get_deploy_command(self, container_type):
+    def get_deploy_command(self, ptype):
         """
         In the deploy phase of dryccfile
         Return the kubernetes "container command" to be sent off to the scheduler.
         """
         return self.build.dryccfile.get(
-            'deploy', {}).get(container_type, {}).get('command', [])
+            'deploy', {}).get(ptype, {}).get('command', [])
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
@@ -152,21 +152,21 @@ def new(self, user, config, build, summary=None):
             build=build, version=new_version, summary=summary
         )
 
-    def get_port(self, procfile_type):
+    def get_port(self, ptype):
         """
         Get application port for a given release. If pulling from private registry
         then use default port or read from ENV var, otherwise attempt to pull from
         the container image
         """
         return int(self.config.typed_values.get(
-            procfile_type, {}).get(
+            ptype, {}).get(
                 'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
 
-    def deploy(self, procfile_types=None, force_deploy=False):
+    def deploy(self, ptypes=None, force_deploy=False):
         lock = DeployLock(self.app.pk)
-        if not lock.acquire(procfile_types, force=force_deploy):
+        if not lock.acquire(ptypes, force=force_deploy):
             raise DryccException('there is an executing pipeline, please wait or force deploy')
-        run_pipeline.delay(self, procfile_types, force_deploy)
+        run_pipeline.delay(self, ptypes, force_deploy)
 
     def previous(self):
         """
@@ -192,7 +192,7 @@ def previous(self):
             prev_release = None
         return prev_release
 
-    def rollback(self, user, procfile_types=None, version=None):
+    def rollback(self, user, ptypes=None, version=None):
         try:
             # if no version is provided then grab version from object
             version = (self.version - 1) if version is None else int(version)
@@ -212,7 +212,7 @@ def rollback(self, user, procfile_types=None, version=None):
                 summary="{} rolled back to v{}".format(user, version),
             )
             if self.build is not None:
-                new_release.deploy(procfile_types, force_deploy=True)
+                new_release.deploy(ptypes, force_deploy=True)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
@@ -231,7 +231,7 @@ def rollback(self, user, procfile_types=None, version=None):
                 new_release.save(update_fields=["state", "failed", "summary", "exception"])
             raise DryccException(str(e)) from e
 
-    def cleanup_old(self, procfile_types=None):
+    def cleanup_old(self, ptypes=None):
         """
         Cleanup any old resources from Kubernetes
 
@@ -245,8 +245,8 @@ def cleanup_old(self, procfile_types=None):
         )
         # base labels
         labels = {'heritage': 'drycc'}
-        if procfile_types is not None:
-            labels['type__in'] = procfile_types
+        if ptypes is not None:
+            labels['type__in'] = ptypes
         # Cleanup controllers
         replica_sets_removal = []
         replica_sets = self.scheduler().rs.get(self.app.id, labels=labels).json()['items']
@@ -270,16 +270,16 @@ def cleanup_old(self, procfile_types=None):
             )
         # this is RC related
         for version_name in replica_sets_removal:
-            self._delete_release_in_scheduler(self.app.id, procfile_types, version_name)
+            self._delete_release_in_scheduler(self.app.id, ptypes, version_name)
         # handle Deployments specific cleanups
-        self._cleanup_deployment_secrets_and_configs(self.app.id, procfile_types)
+        self._cleanup_deployment_secrets_and_configs(self.app.id, ptypes)
         # Remove stray pods
-        self._cleanup_stray_pods(self.app.id, procfile_types, self.version_name)
+        self._cleanup_stray_pods(self.app.id, ptypes, self.version_name)
 
-    def _cleanup_stray_pods(self, namespace, procfile_types, latest_version_name):
+    def _cleanup_stray_pods(self, namespace, ptypes, latest_version_name):
         labels = {'heritage': 'drycc'}
-        if procfile_types is not None:
-            labels['type__in'] = procfile_types
+        if ptypes is not None:
+            labels['type__in'] = ptypes
         pods = self.scheduler().pod.get(namespace, labels=labels).json()['items']
         if not pods:
             pods = []
@@ -299,7 +299,7 @@ def _cleanup_stray_pods(self, namespace, procfile_types, latest_version_name):
                 if e.response.status_code == 404:
                     continue
 
-    def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None):
+    def _cleanup_deployment_secrets_and_configs(self, namespace, ptypes=None):
         """
         Clean up any environment secrets (and in the future ConfigMaps) that
         are tied to a release Deployments no longer track
@@ -332,8 +332,8 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
             # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
             'version__notin': version_names
         }
-        if procfile_types is not None:
-            labels['type__in'] = procfile_types
+        if ptypes is not None:
+            labels['type__in'] = ptypes
         self.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
         secrets = self.scheduler().secret.get(namespace, labels=labels).json()['items']
         if not secrets:
@@ -341,7 +341,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, procfile_types=None
         for secret in secrets:
             self.scheduler().secret.delete(namespace, secret['metadata']['name'])
 
-    def _delete_release_in_scheduler(self, namespace, procfile_types, version_name):
+    def _delete_release_in_scheduler(self, namespace, ptypes, version_name):
         """
         Deletes a specific release in k8s based on ReplicationController
 
@@ -353,8 +353,8 @@ def _delete_release_in_scheduler(self, namespace, procfile_types, version_name):
             'app': namespace,
             'version': version_name
         }
-        if procfile_types is not None:
-            labels['type__in'] = procfile_types
+        if ptypes is not None:
+            labels['type__in'] = ptypes
         # see if the app config has deploy timeout preference, otherwise use global
         timeout = self.config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)
 
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index 82758dd54..9ad5219fb 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -6,7 +6,7 @@
 from api.utils import validate_json
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
-from .base import AuditedModel, PROCFILE_TYPE_MAX_LENGTH
+from .base import AuditedModel, PTYPE_MAX_LENGTH
 
 
 User = get_user_model()
@@ -33,11 +33,11 @@ class Service(AuditedModel):
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     ports = models.JSONField(
         default=list, validators=[partial(validate_json, schema=service_ports_schema)])
-    procfile_type = models.CharField(max_length=PROCFILE_TYPE_MAX_LENGTH)
+    ptype = models.CharField(max_length=PTYPE_MAX_LENGTH)
 
     class Meta:
         get_latest_by = 'created'
-        unique_together = (('app', 'procfile_type'), )
+        unique_together = (('app', 'ptype'), )
         ordering = ['-created']
 
     def __str__(self):
@@ -53,7 +53,7 @@ def as_dict(self):
         return {
             "domain": self.domain,
             "ports": self.ports,
-            "procfile_type": self.procfile_type,
+            "ptype": self.ptype,
         }
 
     def port_name(self, port, protocol):
@@ -120,10 +120,10 @@ def _namespace(self):
         return self.app.id
 
     def _svc_name(self):
-        if self.procfile_type == 'web':
+        if self.ptype == 'web':
             svc_name = self.app.id
         else:
-            svc_name = "{}-{}".format(self.app.id, self.procfile_type)
+            svc_name = "{}-{}".format(self.app.id, self.ptype)
         return svc_name
 
     def refresh_k8s_svc(self):
@@ -136,12 +136,12 @@ def refresh_k8s_svc(self):
                 self.scheduler().svc.patch(namespace, svc_name, **{
                     "ports": self.ports,
                     "version": data["metadata"]["resourceVersion"],
-                    "procfile_type": self.procfile_type,
+                    "ptype": self.ptype,
                 })
             except KubeException:
                 self.scheduler().svc.create(namespace, svc_name, **{
                     "ports": self.ports,
-                    "procfile_type": self.procfile_type,
+                    "ptype": self.ptype,
                 })
         except KubeException as e:
             raise ServiceUnavailable('Kubernetes service could not be created') from e
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 4d2bb4edd..7b35a8bd2 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -161,9 +161,9 @@ def query_network_flow(namespaces: Iterator[str],
         yield from cursor
 
 
-def query_container_count(namespace: str, container_type: str, start: int, stop: int) -> int:
+def query_container_count(namespace: str, ptype: str, start: int, stop: int) -> int:
     with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, container_type)
+        container_name = "%s-%s" % (namespace, ptype)
         sql = query_container_count_sql_tpl.format(
             namespace=namespace, container_name=container_name, start=start, stop=stop)
         cursor.execute(sql)
@@ -171,11 +171,11 @@ def query_container_count(namespace: str, container_type: str, start: int, stop:
         return row[0] if row else 0
 
 
-def query_cpu_usage(namespace: str, container_type: str,
+def query_cpu_usage(namespace: str, ptype: str,
                     start: int, stop: int, every: str
                     ) -> Iterator[tuple[str, str, str, int, int, int]]:
     with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, container_type)
+        container_name = "%s-%s" % (namespace, ptype)
         sql = query_cpu_usage_sql_tpl.format(
             namespace=namespace, container_name=container_name,
             start=start, stop=stop, every=every)
@@ -183,11 +183,11 @@ def query_cpu_usage(namespace: str, container_type: str,
         yield from cursor
 
 
-def query_memory_usage(namespace: str, container_type: str,
+def query_memory_usage(namespace: str, ptype: str,
                        start: int, stop: int, every: str
                        ) -> Iterator[tuple[str, str, str, int, int, int]]:
     with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, container_type)
+        container_name = "%s-%s" % (namespace, ptype)
         sql = query_memory_usage_sql_tpl.format(
             namespace=namespace, container_name=container_name,
             start=start, stop=stop, every=every)
@@ -195,11 +195,11 @@ def query_memory_usage(namespace: str, container_type: str,
         yield from cursor
 
 
-def query_network_usage(namespace: str, container_type: str,
+def query_network_usage(namespace: str, ptype: str,
                         start: int, stop: int, every: str
                         ) -> Iterator[tuple[str, str, int, int, int]]:
     with closing(connections['monitor'].cursor()) as cursor:
-        pod_name_prefix = "%s-%s" % (namespace, container_type)
+        pod_name_prefix = "%s-%s" % (namespace, ptype)
         sql = query_network_usage_sql_tpl.format(
             namespace=namespace, pod_name_prefix=pod_name_prefix,
             start=start, stop=stop, every=every)
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index a3e018fd6..b4662024f 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -17,7 +17,7 @@
 from api import models
 from api.utils import validate_json
 from api.exceptions import DryccException
-from api.models.base import PROCFILE_TYPE_MIN_LENGTH, PROCFILE_TYPE_MAX_LENGTH
+from api.models.base import PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 from .schemas import rules
 from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
@@ -61,13 +61,13 @@
 HEALTHCHECK_MISMATCH_MSG = "Healthcheck pattern: %s" % HEALTHCHECK_MATCH.pattern
 
 
-def validate_procfile_type(value):
+def validate_ptype(value):
     if not re.match(PROCTYPE_MATCH, value):
         raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-    if len(value) < PROCFILE_TYPE_MIN_LENGTH or len(value) > PROCFILE_TYPE_MAX_LENGTH:
+    if len(value) < PTYPE_MIN_LENGTH or len(value) > PTYPE_MAX_LENGTH:
         raise serializers.ValidationError(
-            "The length of procfile_type must be between {} and {}".format(
-                PROCFILE_TYPE_MIN_LENGTH, PROCFILE_TYPE_MAX_LENGTH))
+            "The length of ptype must be between {} and {}".format(
+                PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH))
     return value
 
 
@@ -204,17 +204,17 @@ def validate_procfile(data):
         for key, value in data.items():
             if value is None or value == "":
                 raise serializers.ValidationError("Command can't be empty for process type")
-            validate_procfile_type(key)
+            validate_ptype(key)
         return data
 
     @staticmethod
     def validate_dryccfile(data):
         if data:
             validate_json(data, DRYCCFILE_SCHEMA, serializers.ValidationError)
-            procfile_types = set().union(data.get("deploy", {}).keys())
-            procfile_types = procfile_types.union(data.get("build", {}).get("docker", {}).keys())
-            for procfile_type in procfile_types:
-                validate_procfile_type(procfile_type)
+            ptypes = set().union(data.get("deploy", {}).keys())
+            ptypes = ptypes.union(data.get("build", {}).get("docker", {}).keys())
+            for ptype in ptypes:
+                validate_ptype(ptype)
         return data
 
 
@@ -285,8 +285,8 @@ def validate_values(data):
 
     @classmethod
     def validate_typed_values(cls, data):
-        for procfile_type, values in data.items():
-            validate_procfile_type(procfile_type)
+        for ptype, values in data.items():
+            validate_ptype(ptype)
             if values is None:  # use NoneType to unset an item
                 continue
             cls.validate_values(values)
@@ -295,8 +295,8 @@ def validate_typed_values(cls, data):
     @staticmethod
     def validate_limits(data):
         req_plan_ids = []
-        for procfile_type, plan_id in data.items():
-            validate_procfile_type(procfile_type)
+        for ptype, plan_id in data.items():
+            validate_ptype(ptype)
             if plan_id is not None:
                 req_plan_ids.append(plan_id)
         plan_ids = [plan.id for plan in models.limit.LimitPlan.objects.filter(
@@ -308,8 +308,8 @@ def validate_limits(data):
 
     @staticmethod
     def validate_termination_grace_period(data):
-        for procfile_type, value in data.items():
-            validate_procfile_type(procfile_type)
+        for ptype, value in data.items():
+            validate_ptype(ptype)
             if value is None:  # use NoneType to unset an item
                 continue
             timeout = re.match(TERMINATION_GRACE_PERIOD_MATCH, str(value))
@@ -321,8 +321,8 @@ def validate_termination_grace_period(data):
 
     @staticmethod
     def validate_tags(data):
-        for procfile_type, values in data.items():
-            validate_procfile_type(procfile_type)
+        for ptype, values in data.items():
+            validate_ptype(ptype)
             for key, value in values.items():
                 if value is None:  # use NoneType to unset an item
                     continue
@@ -368,8 +368,8 @@ def validate_registry(data):
 
     @staticmethod
     def validate_healthcheck(data):
-        for procfile_type, healthcheck in data.items():
-            validate_procfile_type(procfile_type)
+        for ptype, healthcheck in data.items():
+            validate_ptype(ptype)
             if healthcheck is None:
                 continue
             for key, value in healthcheck.items():
@@ -413,7 +413,7 @@ class DomainSerializer(serializers.ModelSerializer):
     class Meta:
         """Metadata options for a :class:`DomainSerializer`."""
         model = models.domain.Domain
-        fields = ['owner', 'created', 'updated', 'app', 'domain', 'procfile_type']
+        fields = ['owner', 'created', 'updated', 'app', 'domain', 'ptype']
         read_only_fields = ['uuid']
 
     @staticmethod
@@ -463,7 +463,7 @@ def ToACE(x): return idna.alabel(x).decode("utf-8", "strict")
 
         return aceValue
 
-    validate_procfile_type = staticmethod(validate_procfile_type)
+    validate_ptype = staticmethod(validate_ptype)
 
 
 class ServiceSerializer(serializers.ModelSerializer):
@@ -474,12 +474,12 @@ class ServiceSerializer(serializers.ModelSerializer):
     port = serializers.IntegerField(required=True)
     protocol = serializers.CharField(required=True)
     target_port = serializers.IntegerField(default=DEFAULT_CONTAINER_PORT)
-    procfile_type = serializers.CharField(required=True)
+    ptype = serializers.CharField(required=True)
 
     class Meta:
         """Metadata options for a :class:`ServiceSerializer`."""
         model = models.service.Service
-        fields = ['owner', 'created', 'updated', 'app', 'procfile_type']
+        fields = ['owner', 'created', 'updated', 'app', 'ptype']
         read_only_fields = ['uuid']
 
     @staticmethod
@@ -500,7 +500,7 @@ def validate_protocol(value):
     def validate_target_port(cls, value):
         return cls.validate_port(value)
 
-    validate_procfile_type = staticmethod(validate_procfile_type)
+    validate_ptype = staticmethod(validate_ptype)
 
 
 class CertificateSerializer(serializers.ModelSerializer):
@@ -547,7 +547,7 @@ def to_representation(self, obj):
         return obj
 
 
-class PtypesSerializer(serializers.BaseSerializer):
+class PtypeSerializer(serializers.BaseSerializer):
     name = serializers.CharField(required=False)
     release = serializers.CharField()
     ready = serializers.CharField()
@@ -628,7 +628,7 @@ def validate_path(data):
         logger.debug(f"mount validate_path data: {data}")
         new_data = {}
         for key, value in data.items():
-            validate_procfile_type(key)
+            validate_ptype(key)
             if value is None:  # use NoneType to unset an item
                 new_data[key] = value
                 continue
@@ -734,7 +734,7 @@ def validate_protocol(value):
             raise serializers.ValidationError(GATEWAY_PROTOCOL_MISMATCH_MSG)
         return value
 
-    validate_procfile_type = staticmethod(validate_procfile_type)
+    validate_ptype = staticmethod(validate_ptype)
 
 
 class RouteSerializer(serializers.Serializer):
@@ -743,7 +743,7 @@ class RouteSerializer(serializers.Serializer):
     kind = serializers.CharField(max_length=15, required=False)
     name = serializers.CharField(max_length=63, required=True)
     port = serializers.IntegerField()
-    procfile_type = serializers.CharField(max_length=63, required=True)
+    ptype = serializers.CharField(max_length=63, required=True)
     rules = serializers.JSONField(required=False)
     parent_refs = serializers.JSONField(required=False)
 
@@ -761,7 +761,7 @@ def validate_kind(value):
             raise serializers.ValidationError(ROUTE_PROTOCOL_MISMATCH_MSG)
         return value
 
-    validate_procfile_type = staticmethod(validate_procfile_type)
+    validate_ptype = staticmethod(validate_ptype)
 
     def validate_rules(self, value):
         kind = getattr(self, "initial_data", getattr(self, "data", {})).get("kind", None)
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index fc127fe05..7c1bf8395 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -182,7 +182,7 @@ def gateway_changed_handle(
 def service_changed_handle(
         sender, instance: Service, created=False, update_fields=None, **kwargs):
     if kwargs['signal'] == post_delete:
-        instance.app.route_set.filter(procfile_type=instance.procfile_type).delete()
+        instance.app.route_set.filter(ptype=instance.ptype).delete()
 
 
 @receiver(signal=[post_save, post_delete], sender=Domain)
diff --git a/rootfs/api/tests/__init__.py b/rootfs/api/tests/__init__.py
index c185fdd23..23ecade9e 100644
--- a/rootfs/api/tests/__init__.py
+++ b/rootfs/api/tests/__init__.py
@@ -81,16 +81,16 @@ def get_or_create_token(self, user):
         )
         return token.key
 
-    def assertPodContains(self, pods, app_id, procfile_type, version, state="up"):
+    def assertPodContains(self, pods, app_id, ptype, version, state="up"):
         for pod in pods:
-            if (pod["type"] == procfile_type and
+            if (pod["type"] == ptype and
                     pod["release"] == version and pod["state"] == state):
-                pod_name = app_id + '-%s-[0-9]{1,10}-[a-z0-9]{5}' % procfile_type
+                pod_name = app_id + '-%s-[0-9]{1,10}-[a-z0-9]{5}' % ptype
                 self.assertRegex(pod['name'], pod_name)
                 return
         raise ValueError(
-            "pod not contains: procfile_type={}, version={}, state={}".format(
-                procfile_type, version, state
+            "pod not contains: ptype={}, version={}, state={}".format(
+                ptype, version, state
             )
         )
 
diff --git a/rootfs/api/tests/test_admissions.py b/rootfs/api/tests/test_admissions.py
index bcd4cbd90..20b2d701c 100644
--- a/rootfs/api/tests/test_admissions.py
+++ b/rootfs/api/tests/test_admissions.py
@@ -53,14 +53,14 @@ def tearDown(self):
 
     def test_admissions_scale(self, requests_mock):
         app_id = self.create_app("myapp")
-        for case, (admission_class, procfile_type, scale) in SCALE_TEST_CASES:
+        for case, (admission_class, ptype, scale) in SCALE_TEST_CASES:
             with open(os.path.join(TEST_ROOT, "admissions", case)) as f:
                 request = json.loads(f.read())["request"]
                 handler = admission_class()
                 self.assertEqual(handler.detect(request), True)
                 self.assertEqual(handler.handle(request), True)
                 app = App.objects.get(id=app_id)
-                self.assertEqual(app.structure.get(procfile_type, 0), scale)
+                self.assertEqual(app.structure.get(ptype, 0), scale)
 
     def test_admissions_service_catalog(self, requests_mock):
         app_id = self.create_app("myapp")
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 7221e8164..c92f4d215 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -16,7 +16,7 @@
 from django.test.utils import override_settings
 
 from api.models.app import App, app_permission_registry
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.config import Config
 from scheduler import KubeException, KubeHTTPException
 
@@ -587,14 +587,14 @@ def test_build_env_vars(self, mock_requests):
         app = App.objects.create(owner=self.user)
         # Make sure an exception is raised when calling without a build available
         with self.assertRaises(DryccException):
-            app._build_env_vars(app.release_set.latest(), PROCFILE_TYPE_WEB)
+            app._build_env_vars(app.release_set.latest(), PTYPE_WEB)
         data = {'image': 'autotest/example', 'stack': 'heroku-18'}
         url = f"/v2/apps/{app.id}/builds"
         response = self.client.post(url, data)
         self.assertEqual(response.status_code, 201, response.data)
         time_created = app.release_set.latest().created
         self.assertEqual(
-            app._build_env_vars(app.release_set.latest(), PROCFILE_TYPE_WEB),
+            app._build_env_vars(app.release_set.latest(), PTYPE_WEB),
             {
                 'DRYCC_APP': app.id,
                 'WORKFLOW_RELEASE': 'v2',
@@ -609,7 +609,7 @@ def test_build_env_vars(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         time_created = app.release_set.latest().created
         self.assertEqual(
-            app._build_env_vars(app.release_set.latest(), PROCFILE_TYPE_WEB),
+            app._build_env_vars(app.release_set.latest(), PTYPE_WEB),
             {
                 'DRYCC_APP': app.id,
                 'WORKFLOW_RELEASE': 'v3',
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 383eb9b2f..7993bacb9 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -12,7 +12,7 @@
 
 from api.models.build import Build
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from scheduler import KubeException
 
 from api.tests import adapter, DryccTransactionTestCase
@@ -508,7 +508,7 @@ def test_build_image_in_registry(self, mock_requests):
 
         build = Build.objects.get(uuid=response.data['uuid'])
         release = build.app.release_set.latest()
-        self.assertEqual(release.get_deploy_image(PROCFILE_TYPE_WEB), image)
+        self.assertEqual(release.get_deploy_image(PTYPE_WEB), image)
 
         # post an image as a build using registry hostname + port
         url = f"/v2/apps/{app_id}/builds"
@@ -519,7 +519,7 @@ def test_build_image_in_registry(self, mock_requests):
 
         build = Build.objects.get(uuid=response.data['uuid'])
         release = build.app.release_set.latest()
-        self.assertEqual(release.get_deploy_image(PROCFILE_TYPE_WEB), image)
+        self.assertEqual(release.get_deploy_image(PTYPE_WEB), image)
 
     def test_build_image_in_registry_with_auth(self, mock_requests):
         """add authentication to the build"""
diff --git a/rootfs/api/tests/test_certificate_use_case_1.py b/rootfs/api/tests/test_certificate_use_case_1.py
index 725bba70c..ea6d5a64a 100644
--- a/rootfs/api/tests/test_certificate_use_case_1.py
+++ b/rootfs/api/tests/test_certificate_use_case_1.py
@@ -2,7 +2,7 @@
 from django.core.cache import cache
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -27,7 +27,7 @@ def setUp(self):
         self.app = App.objects.create(owner=self.user, id='test-app-use-case-1')
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domain = Domain.objects.create(
-            owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB)
+            owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB)
         self.name = 'foo-com'  # certificate name
 
         with open('{}/certs/{}.key'.format(TEST_ROOT, self.domain)) as f:
diff --git a/rootfs/api/tests/test_certificate_use_case_2.py b/rootfs/api/tests/test_certificate_use_case_2.py
index 646aaa47b..7f808f890 100644
--- a/rootfs/api/tests/test_certificate_use_case_2.py
+++ b/rootfs/api/tests/test_certificate_use_case_2.py
@@ -2,7 +2,7 @@
 from django.core.cache import cache
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -28,9 +28,9 @@ def setUp(self):
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             'foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         # only foo.com has a cert
diff --git a/rootfs/api/tests/test_certificate_use_case_3.py b/rootfs/api/tests/test_certificate_use_case_3.py
index 4163dcd92..9d17a2f97 100644
--- a/rootfs/api/tests/test_certificate_use_case_3.py
+++ b/rootfs/api/tests/test_certificate_use_case_3.py
@@ -2,7 +2,7 @@
 from django.core.cache import cache
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -28,9 +28,9 @@ def setUp(self):
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             'foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
diff --git a/rootfs/api/tests/test_certificate_use_case_4.py b/rootfs/api/tests/test_certificate_use_case_4.py
index 4ed65b506..9bfd62c6f 100644
--- a/rootfs/api/tests/test_certificate_use_case_4.py
+++ b/rootfs/api/tests/test_certificate_use_case_4.py
@@ -2,7 +2,7 @@
 from django.core.cache import cache
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -29,11 +29,11 @@ def setUp(self):
         self.domains = {
             '*.foo.com': Domain.objects.create(
                 owner=self.user, app=self.app, domain='*.foo.com',
-                procfile_type=PROCFILE_TYPE_WEB),
+                ptype=PTYPE_WEB),
             'foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
diff --git a/rootfs/api/tests/test_certificate_use_case_5.py b/rootfs/api/tests/test_certificate_use_case_5.py
index a69e2809c..34c331e3c 100644
--- a/rootfs/api/tests/test_certificate_use_case_5.py
+++ b/rootfs/api/tests/test_certificate_use_case_5.py
@@ -2,7 +2,7 @@
 from django.core.cache import cache
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -28,13 +28,13 @@ def setUp(self):
         self.url = f'/v2/apps/{self.app.id}/certs'
         # Done out of scope as it gets the same cert as the wildcard
         Domain.objects.create(
-            owner=self.user, app=self.app, domain='foo.com', procfile_type=PROCFILE_TYPE_WEB)
+            owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB)
         self.domains = {
             '*.foo.com': Domain.objects.create(
                 owner=self.user, app=self.app, domain='*.foo.com',
-                procfile_type=PROCFILE_TYPE_WEB),
+                ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', procfile_type=PROCFILE_TYPE_WEB),
+                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 4478b2c3d..996a002d3 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -13,7 +13,7 @@
 from unittest import mock
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_RUN, PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_RUN, PTYPE_WEB
 from api.models.config import Config
 from api.serializers import CONFIG_LIMITS_MISMATCH_MSG
 from api.models.build import Build
@@ -145,8 +145,8 @@ def test_response_data(self, mock_requests):
             'app': app_id,
             'values': {'PORT': '5000'},
             'limits': {
-                PROCFILE_TYPE_RUN: 'std1.large.c1m1',
-                PROCFILE_TYPE_WEB: 'std1.large.c1m1'
+                PTYPE_RUN: 'std1.large.c1m1',
+                PTYPE_WEB: 'std1.large.c1m1'
             },
             'tags': {},
             'registry': {}
@@ -162,7 +162,7 @@ def test_response_data_types_converted(self, mock_requests):
         body = {
             'values': json.dumps({'PORT': 5000}),
             'limits': {
-                PROCFILE_TYPE_WEB: 'std1.large.c1m2',
+                PTYPE_WEB: 'std1.large.c1m2',
             }
         }
         response = self.client.post(url, body)
@@ -177,15 +177,15 @@ def test_response_data_types_converted(self, mock_requests):
             'app': app_id,
             'values': {'PORT': '5000'},
             'limits': {
-                PROCFILE_TYPE_RUN: 'std1.large.c1m1',
-                PROCFILE_TYPE_WEB: 'std1.large.c1m2'
+                PTYPE_RUN: 'std1.large.c1m1',
+                PTYPE_WEB: 'std1.large.c1m2'
             },
             'tags': {},
             'registry': {}
         }
         self.assertEqual(response.data, expected | response.data)
 
-        body = {'limits': {PROCFILE_TYPE_WEB: "not-exist"}}
+        body = {'limits': {PTYPE_WEB: "not-exist"}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
         self.assertEqual(
@@ -405,8 +405,8 @@ def test_unset_limits(self, mock_requests):
             'values': json.dumps({'PORT': 5000}),
             'limits': {
                 "task": 'std1.large.c2m4',
-                PROCFILE_TYPE_RUN: 'std1.large.c2m4',
-                PROCFILE_TYPE_WEB: 'std1.large.c2m4',
+                PTYPE_RUN: 'std1.large.c2m4',
+                PTYPE_WEB: 'std1.large.c2m4',
             },
         }
         response = self.client.post(url, body)
@@ -425,7 +425,7 @@ def test_unset_limits(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(
             response.data["limits"],
-            {PROCFILE_TYPE_RUN: 'std1.large.c2m4', PROCFILE_TYPE_WEB: 'std1.large.c2m4'},
+            {PTYPE_RUN: 'std1.large.c2m4', PTYPE_WEB: 'std1.large.c2m4'},
         )
 
     def test_unset_limits_error(self, mock_requests):
@@ -435,8 +435,8 @@ def test_unset_limits_error(self, mock_requests):
             'values': json.dumps({'PORT': 5000}),
             'limits': {
                 "task": 'std1.large.c2m4',
-                PROCFILE_TYPE_RUN: 'std1.large.c2m4',
-                PROCFILE_TYPE_WEB: 'std1.large.c2m4',
+                PTYPE_RUN: 'std1.large.c2m4',
+                PTYPE_WEB: 'std1.large.c2m4',
             },
         }
         response = self.client.post(url, body)
@@ -481,7 +481,7 @@ def test_unset_limits_error(self, mock_requests):
 
         body = {
             'limits': {
-                  PROCFILE_TYPE_WEB: None,
+                  PTYPE_WEB: None,
             },
         }
         response = self.client.post(url, body)
@@ -507,8 +507,8 @@ def test_measure_config(self, *args, **kwargs):
         body = {
             'values': json.dumps({'PORT': 5000}),
             'limits': {
-                  PROCFILE_TYPE_RUN: 'std1.large.c2m4',
-                  PROCFILE_TYPE_WEB: 'std1.large.c2m4',
+                  PTYPE_RUN: 'std1.large.c2m4',
+                  PTYPE_WEB: 'std1.large.c2m4',
             },
         }
         response = self.client.post(url, body)
@@ -546,8 +546,8 @@ def test_set_config_limits_run(self, *args, **kwargs):
         body = {
             'values': json.dumps({'PORT': 5000}),
             'limits': {
-                  PROCFILE_TYPE_RUN: 'std1.large.c2m4',
-                  PROCFILE_TYPE_WEB: 'std1.large.c2m4',
+                  PTYPE_RUN: 'std1.large.c2m4',
+                  PTYPE_WEB: 'std1.large.c2m4',
             },
         }
         url = f"/v2/apps/{app_id}/config"
diff --git a/rootfs/api/tests/test_domain.py b/rootfs/api/tests/test_domain.py
index c771ce989..7b310ca8d 100644
--- a/rootfs/api/tests/test_domain.py
+++ b/rootfs/api/tests/test_domain.py
@@ -8,7 +8,7 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.domain import Domain
 from api.tests import DryccTestCase
 
@@ -39,13 +39,13 @@ def test_response_data(self):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': 'test-domain.example.com', 'procfile_type': PROCFILE_TYPE_WEB}
+            {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB}
         )
         self.assertEqual(response.status_code, 201, response.data)
 
         for key in response.data:
             self.assertIn(
-                key, ['uuid', 'owner', 'created', 'updated', 'app', 'domain', 'procfile_type'])
+                key, ['uuid', 'owner', 'created', 'updated', 'app', 'domain', 'ptype'])
 
         expected = {
             'owner': self.user.username,
@@ -61,7 +61,7 @@ def test_strip_dot(self):
         url = '/v2/apps/{app_id}/domains'.format(app_id=self.app_id)
 
         # Create
-        response = self.client.post(url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+        response = self.client.post(url, {'domain': domain, 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 201, msg)
 
         # Fetch
@@ -99,7 +99,7 @@ def test_manage_idn_domain(self):
 
             # Create
             response = self.client.post(
-                url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, msg)
 
             # Fetch
@@ -111,13 +111,13 @@ def test_manage_idn_domain(self):
             # Verify creation failure for same domain with different encoding
             if ace_domain != domain:
                 response = self.client.post(
-                    url, {'domain': ace_domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                    url, {'domain': ace_domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 400, msg)
 
             # Verify creation failure for same domain with different encoding
             if unicode_domain != domain:
                 response = self.client.post(
-                    url, {'domain': unicode_domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                    url, {'domain': unicode_domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 400, msg)
 
             # Delete
@@ -139,7 +139,7 @@ def test_manage_idn_domain(self):
             if ace_domain != domain:
                 # Create
                 response = self.client.post(
-                    url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                    url, {'domain': domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 201, msg)
 
                 # Fetch
@@ -167,7 +167,7 @@ def test_manage_idn_domain(self):
             if unicode_domain != domain:
                 # Create
                 response = self.client.post(
-                    url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                    url, {'domain': domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 201, msg)
 
                 # Fetch
@@ -213,7 +213,7 @@ def test_manage_domain(self):
 
             # Create
             response = self.client.post(
-                url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, msg)
 
             # Fetch
@@ -253,7 +253,7 @@ def test_delete_domain_does_not_remove_latest(self):
         ]
         for domain in test_domains:
             response = self.client.post(
-                url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, response.data)
 
         url = '/v2/apps/{app_id}/domains/{domain}'.format(domain=test_domains[0],
@@ -272,7 +272,7 @@ def test_manage_domain_invalid_app(self):
         # Create domain
         url = '/v2/apps/{app_id}/domains'.format(app_id="this-app-does-not-exist")
         response = self.client.post(
-            url, {'domain': 'test-domain.example.com', 'procfile_type': PROCFILE_TYPE_WEB})
+            url, {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 404)
 
         # verify
@@ -295,7 +295,7 @@ def test_manage_domain_invalid_domain(self):
         for domain in test_domains:
             msg = "failed on \"{}\"".format(domain)
             response = self.client.post(
-                url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 400, msg)
 
     def test_admin_can_add_domains_to_other_apps(self):
@@ -311,7 +311,7 @@ def test_admin_can_add_domains_to_other_apps(self):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         url = '/v2/apps/{}/domains'.format(app_id)
         response = self.client.post(
-            url, {'domain': 'example.drycc.example.com', 'procfile_type': PROCFILE_TYPE_WEB})
+            url, {'domain': 'example.drycc.example.com', 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 201, response.data)
 
     def test_unauthorized_user_cannot_modify_domain(self):
@@ -329,7 +329,7 @@ def test_unauthorized_user_cannot_modify_domain(self):
 
         url = '/v2/apps/{}/domains'.format(app_id)
         response = self.client.post(
-            url, {'domain': 'example.com', 'procfile_type': PROCFILE_TYPE_WEB})
+            url, {'domain': 'example.com', 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 403)
 
     def test_kubernetes_service_failure(self):
@@ -343,5 +343,5 @@ def test_kubernetes_service_failure(self):
             domain = 'foo.com'
             url = '/v2/apps/{}/domains'.format(app_id)
             response = self.client.post(
-                url, {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB})
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_event.py b/rootfs/api/tests/test_event.py
index bc02beded..9e323a4d2 100644
--- a/rootfs/api/tests/test_event.py
+++ b/rootfs/api/tests/test_event.py
@@ -70,5 +70,5 @@ def test_events(self, mock_requests):
 
         # list events of deployment
         url = f"/v2/apps/{app_id}/events"
-        response = self.client.get(url, {"ptype_name": f"{app_id}-web"})
+        response = self.client.get(url, {"ptype": f"{app_id}-web"})
         self.assertEqual(response.status_code, 200, response.data)
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 56fee99ce..b529b260b 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -6,7 +6,7 @@
 from django.core.cache import cache
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.build import Build
 from api.models.gateway import Route
 from api.models.release import Release
@@ -33,7 +33,7 @@ def create_app_with_domain_and_deploy(self):
         app_id = self.create_app()
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': 'test-domain.example.com', 'procfile_type': PROCFILE_TYPE_WEB}
+            {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB}
         )
         self.assertEqual(response.status_code, 201, response.data)
         # check default gateway route
@@ -93,7 +93,7 @@ def create_tls_domain(self, app_id):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': domain, 'procfile_type': PROCFILE_TYPE_WEB}
+            {'domain': domain, 'ptype': PTYPE_WEB}
         )
         self.assertEqual(response.status_code, 201)
         response = self.client.post(
@@ -393,7 +393,7 @@ def test_change_tls(self):
 class RouteTest(BaseGatewayTest):
 
     def create_route(self, app_id, route_name="test-route",
-                     procfile_type="task", kind="HTTPRoute"):
+                     ptype="task", kind="HTTPRoute"):
         # create service
         port = 5000
         response = self.client.post(
@@ -402,7 +402,7 @@ def create_route(self, app_id, route_name="test-route",
                 'port': port,
                 'protocol': 'TCP',
                 'target_port': port,
-                'procfile_type': procfile_type
+                'ptype': ptype
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -411,13 +411,13 @@ def create_route(self, app_id, route_name="test-route",
             '/v2/apps/{}/routes/'.format(app_id),
             {
                 "port": 5000,
-                "procfile_type": procfile_type,
+                "ptype": ptype,
                 "kind": kind,
                 "name": route_name,
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
-        return procfile_type, port, route_name
+        return ptype, port, route_name
 
     def test_create_route(self):
         app_id = self.create_app()
@@ -427,7 +427,7 @@ def test_create_route(self):
             '/v2/apps/{}/routes/'.format(app_id),
             {
                 "port": 5000,
-                "procfile_type": "no-exists",
+                "ptype": "no-exists",
                 "kind": "HTTPRoute",
                 "name": "test-route-1",
             }
@@ -436,10 +436,10 @@ def test_create_route(self):
 
     def test_hostnames(self):
         app_id = self.create_app()
-        procfile_type, port, route_name = self.create_route(app_id)
+        ptype, port, route_name = self.create_route(app_id)
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': "domain1.test.com", 'procfile_type': procfile_type}
+            {'domain': "domain1.test.com", 'ptype': ptype}
         )
         self.assertEqual(response.status_code, 201)
         route = Route.objects.get(app__id=app_id)
@@ -447,7 +447,7 @@ def test_hostnames(self):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': "domain2.test.com", 'procfile_type': "web-1"}
+            {'domain': "domain2.test.com", 'ptype': "web-1"}
         )
         self.assertEqual(response.status_code, 201)
         route = Route.objects.get(app__id=app_id)
@@ -455,7 +455,7 @@ def test_hostnames(self):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': "domain3.test.com", 'procfile_type': procfile_type}
+            {'domain': "domain3.test.com", 'ptype': ptype}
         )
         self.assertEqual(response.status_code, 201)
         route = Route.objects.get(app__id=app_id)
@@ -463,7 +463,7 @@ def test_hostnames(self):
 
     def test_route_attach(self):
         app_id = self.create_app()
-        procfile_type, port, route_name = self.create_route(app_id)
+        ptype, port, route_name = self.create_route(app_id)
         gateway_name_1 = 'bing-gateway-1'
         self.create_gateway(app_id, gateway_name_1, 5000, "HTTP")
         self.client.patch(
@@ -486,7 +486,7 @@ def test_route_attach(self):
         )
         response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
         self.assertEqual(len(response.data["results"][0]["parent_refs"]), 2)
-        return procfile_type, app_id, gateway_name_1, gateway_name_2, port, route_name
+        return ptype, app_id, gateway_name_1, gateway_name_2, port, route_name
 
     def test_route_check_parent_ok(self):
         app_id = self.create_app()
@@ -603,14 +603,14 @@ def test_route_delete(self):
 
     def test_rule_get(self):
         app_id = self.create_app()
-        procfile_type, _, route_name = self.create_route(app_id)
+        ptype, _, route_name = self.create_route(app_id)
         response = self.client.get(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
         )
         expect = [{
             'backendRefs': [{
                 'kind': 'Service',
-                'name': '%s-%s' % (app_id, procfile_type),
+                'name': '%s-%s' % (app_id, ptype),
                 'port': 5000,
                 'weight': 100
             }]
@@ -628,11 +628,11 @@ def test_schemas(self):
 
     def test_rule_set(self):
         app_id = self.create_app()
-        procfile_type, _, route_name = self.create_route(app_id)
+        ptype, _, route_name = self.create_route(app_id)
         expect = [{
             "backendRefs": [{
                 "kind": "Service",
-                "name": "%s-%s" % (app_id, procfile_type),
+                "name": "%s-%s" % (app_id, ptype),
                 "port": 5000,
                 "weight": 100
             }]
@@ -649,7 +649,7 @@ def test_rule_set(self):
         expect = [{
             "backendRefs": [{
                 "kind": "Service",
-                "name": "%s-%s-noexits" % (app_id, procfile_type),
+                "name": "%s-%s-noexits" % (app_id, ptype),
                 "port": 5000,
                 "weight": 100
             }]
diff --git a/rootfs/api/tests/test_procfile.py b/rootfs/api/tests/test_procfile.py
index b8210ecd5..2b2d01423 100644
--- a/rootfs/api/tests/test_procfile.py
+++ b/rootfs/api/tests/test_procfile.py
@@ -1,6 +1,6 @@
 from rest_framework.test import APITestCase
 from rest_framework import serializers
-from api.serializers import validate_procfile_type, BuildSerializer, ServiceSerializer
+from api.serializers import validate_ptype, BuildSerializer, ServiceSerializer
 
 
 class ProcfileTypeTest(APITestCase):
@@ -18,17 +18,17 @@ def assertException(self, method, exception=Exception, *args, **kwargs):
             raise AssertionError(
                 f"Did not throw the expected exception, args: {args}, kwargs: {kwargs}")
 
-    def test_procfile_type_error(self):
-        for procfile_type in ["w", "we", "-a", "we-new-", "w" * 64]:
+    def test_ptype_error(self):
+        for ptype in ["w", "we", "-a", "we-new-", "w" * 64]:
             self.assertException(
-                validate_procfile_type, serializers.ValidationError, procfile_type)
+                validate_ptype, serializers.ValidationError, ptype)
 
-    def test_procfile_type_ok(self):
-        self.assertEqual(validate_procfile_type("web"), "web")
-        self.assertEqual(validate_procfile_type("w" * 63), "w" * 63)
-        self.assertEqual(validate_procfile_type("web-new-be"), "web-new-be")
+    def test_ptype_ok(self):
+        self.assertEqual(validate_ptype("web"), "web")
+        self.assertEqual(validate_ptype("w" * 63), "w" * 63)
+        self.assertEqual(validate_ptype("web-new-be"), "web-new-be")
 
-    def test_dryccfile_procfile_type(self):
+    def test_dryccfile_ptype(self):
         dryccfile_1 = {
             "build": {
                 "docker": {
@@ -60,11 +60,11 @@ def test_dryccfile_procfile_type(self):
         self.assertException(
                 validate_dryccfile, serializers.ValidationError, dryccfile_1)
 
-    def test_staticmethod_procfile_type(self):
-        s_validate_procfile_type = ServiceSerializer().validate_procfile_type
-        for procfile_type in ["w", "we", "-a", "we-new-", "w" * 64]:
+    def test_staticmethod_ptype(self):
+        s_validate_ptype = ServiceSerializer().validate_ptype
+        for ptype in ["w", "we", "-a", "we-new-", "w" * 64]:
             self.assertException(
-                s_validate_procfile_type, serializers.ValidationError, procfile_type)
-        self.assertEqual(s_validate_procfile_type("web"), "web")
-        self.assertEqual(s_validate_procfile_type("w" * 63), "w" * 63)
-        self.assertEqual(s_validate_procfile_type("web-new-be"), "web-new-be")
+                s_validate_ptype, serializers.ValidationError, ptype)
+        self.assertEqual(s_validate_ptype("web"), "web")
+        self.assertEqual(s_validate_ptype("w" * 63), "w" * 63)
+        self.assertEqual(s_validate_ptype("web-new-be"), "web-new-be")
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
index 1c83ec1a7..fcda2092c 100644
--- a/rootfs/api/tests/test_ptypes.py
+++ b/rootfs/api/tests/test_ptypes.py
@@ -68,8 +68,8 @@ def test_ptype(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
-        ptype_name = "no-exists-ptype-name"
-        response = self.client.get(f"/v2/apps/{app_id}/ptypes/{ptype_name}/describe/")
+        ptype = "no-exists-ptype-name"
+        response = self.client.get(f"/v2/apps/{app_id}/ptypes/{ptype}/describe/")
         self.assertEqual(response.status_code, 400, response.data)
 
     def test_restart_ptypes(self, mock_requests):
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index b6eec3d66..ea859dde7 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -11,7 +11,7 @@
 from unittest import mock
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.build import Build
 from api.models.release import Release
 from scheduler import KubeHTTPException
@@ -301,7 +301,7 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(release5.build, release3.build)
         self.assertEqual(release5.config.values, release3.config.values)
         # double-check to see that the current build and config is the same as v3
-        self.assertEqual(release5.get_deploy_image(PROCFILE_TYPE_WEB), 'autotest/example')
+        self.assertEqual(release5.get_deploy_image(PTYPE_WEB), 'autotest/example')
         self.assertEqual(release5.config.values, {'NEW_URL1': 'http://localhost:8080/'})
         # try to rollback to v1 and verify that the rollback failed
         # (v1 is an initial release with no build)
@@ -316,7 +316,7 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(Release.objects.count(), 6)
         release6 = Release.objects.get(app=app, version=6)
-        self.assertEqual(release6.get_deploy_image(PROCFILE_TYPE_WEB), 'autotest/example')
+        self.assertEqual(release6.get_deploy_image(PTYPE_WEB), 'autotest/example')
         self.assertEqual(release6.config.values, {})
 
     def test_release_str(self, mock_requests):
@@ -651,7 +651,7 @@ def test_release_external_registry(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.get_port('web'), 3000)
-        self.assertEqual(release.get_deploy_image(PROCFILE_TYPE_WEB), 'test/autotest/example')
+        self.assertEqual(release.get_deploy_image(PTYPE_WEB), 'test/autotest/example')
 
         url = f'/v2/apps/{app_id}/config'
         body = {'typed_values': json.dumps({"web": {'PORT': '9000'}})}
diff --git a/rootfs/api/tests/test_service.py b/rootfs/api/tests/test_service.py
index 144d1fb65..869c2c201 100644
--- a/rootfs/api/tests/test_service.py
+++ b/rootfs/api/tests/test_service.py
@@ -41,7 +41,7 @@ def test_service_basic_ops(self):
                 'port': 5000,
                 'protocol': 'UDP',
                 'target_port': 5000,
-                'procfile_type': 'test'
+                'ptype': 'test'
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -58,7 +58,7 @@ def test_service_basic_ops(self):
                 'protocol': 'UDP',
                 'targetPort': 5000,
             }],
-            "procfile_type": "test"
+            "ptype": "test"
         }
         self.assertEqual(response.data['services'][0], expected0 | response.data['services'][0])
         # port is occupied
@@ -68,7 +68,7 @@ def test_service_basic_ops(self):
                 'port': 5000,
                 'protocol': 'UDP',
                 'target_port': 5000,
-                'procfile_type': 'test'
+                'ptype': 'test'
             }
         )
         self.assertEqual(response.status_code, 400, response.data)
@@ -91,7 +91,7 @@ def test_service_basic_ops(self):
                     'targetPort': 6000,
                 }
             ],
-            "procfile_type": "test"
+            "ptype": "test"
         }
         response = self.client.post(
             '/v2/apps/{}/services'.format(app_id),
@@ -99,7 +99,7 @@ def test_service_basic_ops(self):
                 'port': 6000,
                 'protocol': 'TCP',
                 'target_port': 6000,
-                'procfile_type': 'test'
+                'ptype': 'test'
             }
         )
         self.assertEqual(response.status_code, 204, response.data)
@@ -115,7 +115,7 @@ def test_service_basic_ops(self):
                 'port': 5000,
                 'protocol': 'UDP',
                 'target_port': 5000,
-                'procfile_type': 'test2'
+                'ptype': 'test2'
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -132,27 +132,27 @@ def test_service_basic_ops(self):
                 'protocol': 'UDP',
                 'targetPort': 5000,
             }],
-            "procfile_type": "test2"
+            "ptype": "test2"
         }
         self.assertEqual(response.data['services'][0], expected2 | response.data['services'][0])
         self.assertEqual(response.data['services'][1], expected1 | response.data['services'][1])
         # delete port
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test', "protocol": "TCP", "port": 6000}
+            {'ptype': 'test', "protocol": "TCP", "port": 6000}
         )
         response = self.client.get('/v2/apps/{}/services'.format(app_id))
         self.assertEqual(response.data['services'][1], expected0 | response.data['services'][1])
         # delete 1st
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test', "protocol": "UDP", "port": 5000}
+            {'ptype': 'test', "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 204, response.data)
         # delete 2nd
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test2',  "protocol": "UDP", "port": 5000}
+            {'ptype': 'test2',  "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 204, response.data)
         response = self.client.get('/v2/apps/{}/services'.format(app_id))
@@ -160,6 +160,6 @@ def test_service_basic_ops(self):
         # delete non-existing (1st again)
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test', "protocol": "UDP", "port": 5000}
+            {'ptype': 'test', "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 404, response.data)
diff --git a/rootfs/api/tests/test_tls.py b/rootfs/api/tests/test_tls.py
index bbdcbeed9..3ca2baf0e 100644
--- a/rootfs/api/tests/test_tls.py
+++ b/rootfs/api/tests/test_tls.py
@@ -4,7 +4,7 @@
 from django.contrib.auth import get_user_model
 
 from api.models.app import App
-from api.models.base import PROCFILE_TYPE_WEB
+from api.models.base import PTYPE_WEB
 from api.models.tls import TLS
 from api.tests import adapter, DryccTransactionTestCase
 
@@ -76,7 +76,7 @@ def test_tls_events(self, mock_requests):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': 'test-domain.example.com', 'procfile_type': PROCFILE_TYPE_WEB}
+            {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB}
         )
         self.assertEqual(response.status_code, 201, response.data)
         self.change_certs_auto(app_id, True)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 5948043d3..805d42010 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -59,19 +59,19 @@
     # restart deployment/ptype's pods
     re_path(
         r"^apps/(?P<id>{})/ptypes/restart/?$".format(settings.APP_URL_REGEX),
-        views.PtypesViewSet.as_view({'post': 'restart'})),
+        views.PtypeViewSet.as_view({'post': 'restart'})),
     # scale ptype replcas
     re_path(
         r"^apps/(?P<id>{})/ptypes/scale/?$".format(settings.APP_URL_REGEX),
-        views.PtypesViewSet.as_view({'post': 'scale'})),
+        views.PtypeViewSet.as_view({'post': 'scale'})),
     # list ptypes
     re_path(
         r"^apps/(?P<id>{})/ptypes/?$".format(settings.APP_URL_REGEX),
-        views.PtypesViewSet.as_view({'get': 'list'})),
+        views.PtypeViewSet.as_view({'get': 'list'})),
     # describe ptypes
     re_path(
         r"^apps/(?P<id>{})/ptypes/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),
-        views.PtypesViewSet.as_view({'get': 'describe'})),
+        views.PtypeViewSet.as_view({'get': 'describe'})),
     # list events
     re_path(
         r"^apps/(?P<id>{})/events/?$".format(settings.APP_URL_REGEX),
@@ -215,7 +215,7 @@
         r'^apps/(?P<id>{})/metrics/?$'.format(settings.APP_URL_REGEX),
         views.MetricView.as_view({'get': 'metric'})),
     re_path(
-        r'^apps/(?P<id>{})/metrics/(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)/status/?$'.format(
+        r'^apps/(?P<id>{})/metrics/(?P<ptype>[a-z0-9]+(\-[a-z0-9]+)*)/status/?$'.format(
             settings.APP_URL_REGEX),
         views.MetricView.as_view({'get': 'status'})),
     re_path(
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index b9cc3a384..96f3d57d2 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -226,13 +226,13 @@ def release(self):
 class DeployLock(CacheLock):
 
     def __init__(self, app_id):
-        self.procfile_types_key = f"app:deploy:procfile:types:{app_id}"
+        self.ptypes_key = f"app:deploy:procfile:types:{app_id}"
         super(DeployLock, self).__init__(f"app:deploy:lock:{app_id}")
 
-    def locked(self, procfile_types):
-        value = cache.get(self.procfile_types_key, [])
+    def locked(self, ptypes):
+        value = cache.get(self.ptypes_key, [])
         # None will locks all
-        if procfile_types is None:
+        if ptypes is None:
             if value is None or len(value) > 0:
                 return True
             elif len(value) == 0:
@@ -240,41 +240,41 @@ def locked(self, procfile_types):
         else:
             if value is None:
                 return True
-            for procfile_type in procfile_types:
-                if procfile_type in value:
+            for ptype in ptypes:
+                if ptype in value:
                     return True
         return False
 
-    def acquire(self, procfile_types, force=False):
+    def acquire(self, ptypes, force=False):
         try:
             if super(DeployLock, self).acquire():
-                value = cache.get(self.procfile_types_key, [])
+                value = cache.get(self.ptypes_key, [])
                 # None will locks all
-                if not force and self.locked(procfile_types):
+                if not force and self.locked(ptypes):
                     return False
-                if procfile_types is None:
+                if ptypes is None:
                     value = None
                 else:
-                    value.extend(procfile_types)
+                    value.extend(ptypes)
                     value = list(set(value))
-                cache.set(self.procfile_types_key, value, timeout=3600)
+                cache.set(self.ptypes_key, value, timeout=3600)
                 return True
         finally:
             super(DeployLock, self).release()
         return False
 
-    def release(self, procfile_types):
+    def release(self, ptypes):
         try:
             if super(DeployLock, self).acquire():
-                if procfile_types is None:
-                    cache.delete(self.procfile_types_key)
+                if ptypes is None:
+                    cache.delete(self.ptypes_key)
                 else:
-                    value = cache.get(self.procfile_types_key, [])
+                    value = cache.get(self.ptypes_key, [])
                     if value is None:
                         return
-                    for procfile_type in procfile_types:
-                        value.remove(procfile_type)
-                    cache.set(self.procfile_types_key, value)
+                    for ptype in ptypes:
+                        value.remove(ptype)
+                    cache.set(self.ptypes_key, value)
         finally:
             super(DeployLock, self).release()
 
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index fa95edbb9..a9c024a41 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -398,14 +398,14 @@ def post_save(self, config):
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
-            procfile_types = set()
+            ptypes = set()
             for field, diff in config.diff().items():
-                if field in config.procfile_fields:
+                if field in config.ptype_fields:
                     for value in diff.values():
-                        procfile_types.update(value.keys())
-            # all_diff_fields changed, deploy all.
-            procfile_types = list(procfile_types) if procfile_types else None
-            release.deploy(procfile_types, False)
+                        ptypes.update(value.keys())
+            # allof_fields changed, deploy all.
+            ptypes = list(ptypes) if ptypes else None
+            release.deploy(ptypes, False)
         except BaseException as e:
             config.delete()
             if isinstance(e, AlreadyExists):
@@ -441,9 +441,9 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_200_OK)
 
 
-class PtypesViewSet(AppResourceViewSet):
+class PtypeViewSet(AppResourceViewSet):
     model = models.app.App
-    serializer_class = serializers.PtypesSerializer
+    serializer_class = serializers.PtypeSerializer
 
     def list(self, *args, **kwargs):
         deploys = self.get_app().list_deployments(*args, **kwargs)
@@ -456,18 +456,18 @@ def describe(self, *args, **kwargs):
         deployment_name = kwargs["name"]
         data = self.get_app().describe_deployment(deployment_name)
         if len(data) == 0:
-            raise DryccException("this ptype not found")
+            raise DryccException("this procfile type not found")
         # fake out pagination for now
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
 
     def restart(self, request, *args, **kwargs):
         app = self.get_app()
-        procfile_types = set(
+        ptypes = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        procfile_types = app.check_procfile_types(procfile_types)
-        for procfile_type in set(procfile_types):
-            restart_app.delay(app, **{"type": procfile_type})
+        ptypes = app.check_ptypes(ptypes)
+        for ptype in set(ptypes):
+            restart_app.delay(app, **{"type": ptype})
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def scale(self, request, **kwargs):
@@ -481,13 +481,13 @@ class EventViewSet(AppResourceViewSet):
     serializer_class = serializers.EventSerializer
 
     def list(self, request, **kwargs):
-        ptype_name = request.query_params.get("ptype_name", None)
+        ptype = request.query_params.get("ptype", None)
         pod_name = request.query_params.get("pod_name", None)
-        if not any([ptype_name, pod_name]):
+        if not any([ptype, pod_name]):
             data = []
         else:
             ref_kind, ref_name = ("Pod", pod_name) if pod_name else \
-                ("Deployment", ptype_name)
+                ("Deployment", ptype)
             events = self.get_app().list_events(ref_kind, ref_name)
             data = self.get_serializer(events, many=True).data
         # fake out pagination for now
@@ -539,17 +539,17 @@ def create_or_update(self, request, **kwargs):
         app = self.get_app()
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data.get('protocol'))
-        procfile_type = self.get_serializer().validate_procfile_type(request.data.get(
-            'procfile_type'))
+        ptype = self.get_serializer().validate_ptype(request.data.get(
+            'ptype'))
         target_port = self.get_serializer().validate_target_port(request.data.get('target_port'))
-        service = app.service_set.filter(procfile_type=procfile_type).first()
+        service = app.service_set.filter(ptype=ptype).first()
         if service:
             for item in service.ports:
                 if item["port"] == port:
                     return Response(status=status.HTTP_400_BAD_REQUEST, data={"detail": "port is occupied"})  # noqa
             http_status = status.HTTP_204_NO_CONTENT
         else:
-            service = self.model(owner=app.owner, app=app, procfile_type=procfile_type)
+            service = self.model(owner=app.owner, app=app, ptype=ptype)
             http_status = status.HTTP_201_CREATED
         service.add_port(port, protocol, target_port)
         service.save()
@@ -558,9 +558,9 @@ def create_or_update(self, request, **kwargs):
     def delete(self, request, **kwargs):
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data.get('protocol'))
-        procfile_type = self.get_serializer().validate_procfile_type(
-            request.data.get('procfile_type'))
-        service = get_object_or_404(self.get_queryset(**kwargs), procfile_type=procfile_type)
+        ptype = self.get_serializer().validate_ptype(
+            request.data.get('ptype'))
+        service = get_object_or_404(self.get_queryset(**kwargs), ptype=ptype)
         removed = service.remove_port(port, protocol)
         if len(service.ports) == 0:
             service.delete()
@@ -620,11 +620,11 @@ def get_object(self, **kwargs):
     def deploy(self, request, **kwargs):
         """Deploy the latest release"""
         latest_release = self.get_app().release_set.latest()
-        force_deploy = request.data.get("force", "false").lower() == "true"
-        procfile_types = set(
+        force_deploy = request.data.get("force", False)
+        ptypes = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        procfile_types = latest_release.app.check_procfile_types(procfile_types)
-        latest_release.deploy(procfile_types, force_deploy)
+        ptypes = latest_release.app.check_ptypes(ptypes)
+        latest_release.deploy(ptypes, force_deploy)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def rollback(self, request, **kwargs):
@@ -633,11 +633,11 @@ def rollback(self, request, **kwargs):
         previous release.
         """
         latest_release = self.get_app().release_set.filter(failed=False).latest()
-        procfile_types = set(
+        ptypes = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        procfile_types = latest_release.app.check_procfile_types(procfile_types)
+        ptypes = latest_release.app.check_ptypes(ptypes)
         new_release = latest_release.rollback(
-            request.user, procfile_types, request.data.get('version', None))
+            request.user, ptypes, request.data.get('version', None))
         response = {'version': new_release.version}
         return Response(response, status=status.HTTP_201_CREATED)
 
@@ -860,7 +860,7 @@ def expand(self, request, **kwargs):
     def destroy(self, request, **kwargs):
         volume = self.get_object()
         app = self.get_app()
-        is_subset = set(volume.path.keys()).issubset(set(app.procfile_types))
+        is_subset = set(volume.path.keys()).issubset(set(app.ptypes))
         if volume.path != {} and is_subset:
             raise DryccException("this volume is mounting")
         volume.delete()
@@ -873,11 +873,10 @@ def path(self, request, *args, **kwargs):
         else:
             path = serializers.VolumeSerializer().validate_path(path)
         volume = self.get_object()
-        container_types = [_ for _ in path.keys()
-                           if _ not in volume.app.procfile_types]
-        if container_types:
+        ptypes = [_ for _ in path.keys() if _ not in volume.app.ptypes]
+        if ptypes:
             raise DryccException("process type {} is not included in procfile".
-                                 format(','.join(container_types)))
+                                 format(','.join(ptypes)))
         if set(path.items()).issubset(set(volume.path.items())):
             raise DryccException("mount path not changed")
         volume.check_path(path)
@@ -1081,8 +1080,8 @@ def create(self, request, *args, **kwargs):
         app = self.get_app()
         port = self.get_serializer().validate_port(request.data.get('port'))
         app_settings = app.appsettings_set.latest()
-        procfile_type = self.get_serializer().validate_procfile_type(
-            request.data.get('procfile_type'))
+        ptype = self.get_serializer().validate_ptype(
+            request.data.get('ptype'))
         kind = self.get_serializer().validate_kind(request.data.get('kind'))
         name = request.data['name']
         route = app.route_set.filter(name=name).first()
@@ -1096,7 +1095,7 @@ def create(self, request, *args, **kwargs):
             name=name,
             port=port,
             routable=app_settings.routable,
-            procfile_type=procfile_type,
+            ptype=ptype,
         )
         route.rules = route.default_rules
         if route.rules and not route.rules[0]["backendRefs"]:
@@ -1169,10 +1168,10 @@ def _get_app(self):
         self.check_object_permissions(self.request, app)
         return app
 
-    def _get_cpus(self, app_id, container_type, start, stop, every):
+    def _get_cpus(self, app_id, ptype, start, stop, every):
         avg_list, max_list = [], []
         for _, _, _, timestamp, max, avg in monitor.query_cpu_usage(
-                app_id, container_type, start, stop, every):
+                app_id, ptype, start, stop, every):
             max_list.append((timestamp, max))
             avg_list.append((timestamp, avg))
         return {
@@ -1180,10 +1179,10 @@ def _get_cpus(self, app_id, container_type, start, stop, every):
             "avg": avg_list,
         }
 
-    def _get_memory(self, app_id, container_type, start, stop, every):
+    def _get_memory(self, app_id, ptype, start, stop, every):
         max_list, avg_list = [], []
         for _, _, _, timestamp, max, avg in monitor.query_memory_usage(
-                app_id, container_type, start, stop, every):
+                app_id, ptype, start, stop, every):
             max_list.append((timestamp, max))
             avg_list.append((timestamp, avg))
         return {
@@ -1191,10 +1190,10 @@ def _get_memory(self, app_id, container_type, start, stop, every):
             "avg": avg_list,
         }
 
-    def _get_networks(self, app_id, container_type, start, stop, every):
+    def _get_networks(self, app_id, ptype, start, stop, every):
         networks = []
         for _, _, timestamp, rx_bytes, tx_bytes in monitor.query_network_usage(
-                app_id, container_type, start, stop, every):
+                app_id, ptype, start, stop, every):
             networks.append((timestamp, rx_bytes, tx_bytes))
         return networks
 
@@ -1209,15 +1208,15 @@ def status(self, request, **kwargs):
             'stop'], data.validated_data["every"]
         return Response({
             "id": app_id,
-            "type": kwargs['type'],
-            "count": monitor.query_container_count(app_id, kwargs['type'], start, stop),
+            "ptype": kwargs['ptype'],
+            "count": monitor.query_container_count(app_id, kwargs['ptype'], start, stop),
             "status": {
                 "cpus": self._get_cpus(
-                    app_id, kwargs['type'], start, stop, every),
+                    app_id, kwargs['ptype'], start, stop, every),
                 "memory": self._get_memory(
-                    app_id, kwargs['type'], start, stop, every),
+                    app_id, kwargs['ptype'], start, stop, every),
                 "networks": self._get_networks(
-                    app_id, kwargs['type'], start, stop, every),
+                    app_id, kwargs['ptype'], start, stop, every),
             }
         })
 
diff --git a/rootfs/scheduler/resources/service.py b/rootfs/scheduler/resources/service.py
index 7a7ae7fe5..f8d5e1745 100644
--- a/rootfs/scheduler/resources/service.py
+++ b/rootfs/scheduler/resources/service.py
@@ -28,8 +28,8 @@ def manifest(self, namespace, name, **kwargs):
         }
         if "version" in kwargs:
             data["metadata"]["resourceVersion"] = kwargs.get("version")
-        if "procfile_type" in kwargs:
-            data["spec"]["selector"]['type'] = kwargs.get("procfile_type")
+        if "ptype" in kwargs:
+            data["spec"]["selector"]['type'] = kwargs.get("ptype")
         return data
 
     def get(self, namespace, name=None, **kwargs):
diff --git a/rootfs/scheduler/tests/test_gateway.py b/rootfs/scheduler/tests/test_gateway.py
index 6d6ec5491..385abb2fa 100644
--- a/rootfs/scheduler/tests/test_gateway.py
+++ b/rootfs/scheduler/tests/test_gateway.py
@@ -75,7 +75,7 @@ def create_http_route(self, name="test-gateway"):
             name,
             name,
             port=5000,
-            procfile_type="web",
+            ptype="web",
             rules={},
             parent_refs={},
         )
@@ -97,7 +97,7 @@ def test_patch_http_route(self):
             "test-gateway",
             "test-gateway",
             port=8000,
-            procfile_type="cmd",
+            ptype="cmd",
             version=1,
             rules=rules,
             parent_refs={},
@@ -124,7 +124,7 @@ def create_tcp_route(self, name="test-tcp-route"):
             name,
             name,
             port=5000,
-            procfile_type="celery",
+            ptype="celery",
             rules={},
             parent_refs={},
         )
@@ -146,7 +146,7 @@ def test_patch_tcp_route(self):
             "test-tcp-route",
             "test-tcp-route",
             port=8000,
-            procfile_type="cmd",
+            ptype="cmd",
             version=1,
             rules=rules,
             parent_refs={},
@@ -173,7 +173,7 @@ def create_udp_route(self, name="test-udp-route"):
             name,
             name,
             port=5000,
-            procfile_type="celery",
+            ptype="celery",
             rules={},
             parent_refs={}
         )
@@ -195,7 +195,7 @@ def test_patch_udp_route(self):
             "test-udp-route",
             "test-udp-route",
             port=8000,
-            procfile_type="cmd",
+            ptype="cmd",
             version=1,
             rules=rules,
             parent_refs={}

From 8e4637c407f943a5d1d792bcdf162397505c1d56 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 3 Sep 2024 17:42:44 +0800
Subject: [PATCH 069/200] chore(pod): modify readiness check

---
 rootfs/scheduler/resources/pod.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 941f1bad6..dc36a383f 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -290,7 +290,7 @@ def _set_health_checks(self, container, env, **kwargs):
                 healthchecks['livenessProbe']['httpGet']['port'] = env.get(
                     'PORT', DEFAULT_CONTAINER_PORT)
             container.update(healthchecks)
-        elif kwargs.get('routable', False):
+        if kwargs.get('routable', False) and healthchecks.get('readinessProbe') is None:
             # If routable, set the default probe
             container.update(
                 self._default_container_readiness_probe(env.get('PORT', DEFAULT_CONTAINER_PORT)))

From 1315cd631ac73b325e51560ade9204b3baee4865 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 4 Sep 2024 16:34:08 +0800
Subject: [PATCH 070/200] chore(controller): services protocol default is TCP

* chore(controller): services protocol default is TCP

* chore(controller): owner or admin operate perms
---
 rootfs/api/serializers/__init__.py |  4 +++-
 rootfs/api/views.py                | 34 ++++++++++++++++++------------
 2 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index b4662024f..796e1657a 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -492,7 +492,9 @@ def validate_port(value):
 
     @staticmethod
     def validate_protocol(value):
-        if not re.match(SERVICE_PROTOCOL_MATCH, value):
+        if value is None or value == "":
+            value = "TCP"
+        elif not re.match(SERVICE_PROTOCOL_MATCH, value):
             raise serializers.ValidationError(SERVICE_PROTOCOL_MISMATCH_MSG)
         return value
 
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index a9c024a41..e898c53ca 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -23,6 +23,7 @@
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
 from rest_framework.parsers import MultiPartParser
+from rest_framework.exceptions import PermissionDenied
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
@@ -392,20 +393,19 @@ class ConfigViewSet(ReleasableViewSet):
     serializer_class = serializers.ConfigSerializer
 
     def post_save(self, config):
-        if not config.app.appsettings_set.latest().autodeploy:
-            return
         latest_release = config.app.release_set.filter(failed=False).latest()
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
-            ptypes = set()
-            for field, diff in config.diff().items():
-                if field in config.ptype_fields:
-                    for value in diff.values():
-                        ptypes.update(value.keys())
-            # allof_fields changed, deploy all.
-            ptypes = list(ptypes) if ptypes else None
-            release.deploy(ptypes, False)
+            if config.app.appsettings_set.latest().autodeploy:
+                ptypes = set()
+                for field, diff in config.diff().items():
+                    if field in config.ptype_fields:
+                        for value in diff.values():
+                            ptypes.update(value.keys())
+                # allof_fields changed, deploy all.
+                ptypes = list(ptypes) if ptypes else None
+                release.deploy(ptypes, False)
         except BaseException as e:
             config.delete()
             if isinstance(e, AlreadyExists):
@@ -774,8 +774,14 @@ def create(self, request, *args, **kwargs):
 class AppPermViewSet(AppResourceViewSet):
     """RESTful views for sharing apps with collaborators."""
 
+    def get_app(self, request):
+        app = get_object_or_404(models.app.App, id=self.kwargs['id'])
+        if not permissions.IsOwnerOrAdmin().has_object_permission(request, self, app):
+            raise PermissionDenied()
+        return app
+
     def list(self, request, **kwargs):
-        app = self.get_app()
+        app = self.get_app(request)
         results = [
             {
                 "app": app.id,
@@ -792,7 +798,7 @@ def list(self, request, **kwargs):
         return Response(data=pagination)
 
     def create(self, request, **kwargs):
-        app = self.get_app()
+        app = self.get_app(request)
         username = request.data.get('username')
         shortnames = set([perm for perm in request.data.get("permissions", "").split(",") if perm])
         all_shortnames = models.app.app_permission_registry.shortnames
@@ -810,7 +816,7 @@ def create(self, request, **kwargs):
         return Response(status=status.HTTP_201_CREATED)
 
     def update(self, request, **kwargs):
-        app = self.get_app()
+        app = self.get_app(request)
         user = get_object_or_404(User, username=kwargs['username'])
         shortnames = set([perm for perm in request.data.get("permissions", "").split(",") if perm])
         all_shortnames = models.app.app_permission_registry.shortnames
@@ -832,7 +838,7 @@ def update(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def destroy(self, request, **kwargs):
-        app = self.get_app()
+        app = self.get_app(request)
         username = kwargs['username']
         user = get_object_or_404(User, username=username)
         for codename in get_user_perms(user, app):

From 52aef5c51f7dce9c55ab9fabfea498e1cc48e8d6 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 5 Sep 2024 11:17:59 +0800
Subject: [PATCH 071/200] chore(migration): remove unused cert

---
 .../migrations/0013_migration_permissions_and_certificates.py   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
index 134574746..7ba105043 100644
--- a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -4,6 +4,7 @@
 from guardian.shortcuts import assign_perm, get_users_with_perms, remove_perm
 from api.models.app import App, VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
 from api.models.domain import Domain
+from api.models.certificate import Certificate
 
 
 def migration_permission(apps, schema_editor):
@@ -27,6 +28,7 @@ def migration_certificate(apps, schema_editor):
                 certificate.save()
                 domain.certificate = certificate
                 domain.save()
+    Certificate.objects.filter(app=None).delete()
 
 
 class Migration(migrations.Migration):

From 6cb5293bc9d107d52503fae7981c3f1517a7344c Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 5 Sep 2024 13:41:58 +0800
Subject: [PATCH 072/200] chore(controller): releases add_condition error
 (#161)

---
 rootfs/api/models/app.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 305d1f595..017b968db 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -295,9 +295,11 @@ def pipeline(self, release, ptypes=None, force_deploy=False):
                 self.deploy(release, ptypes, force_deploy, rollback_on_failure)
             if release.state == "created":
                 release.state = "succeed"
+            ptypes = list(ptypes) if ptypes is not None else ptypes
             release.add_condition(state="succeed", action="pipeline", ptypes=ptypes)
         except Exception as e:
             release.failed, release.state = True, "crashed"
+            ptypes = list(ptypes) if ptypes is not None else ptypes
             release.add_condition(
                 state="crashed", action="pipeline", ptypes=ptypes, exception=str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)

From 3ce26a4bfbb7257b0b11c225fa56cbbe6147f059 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 6 Sep 2024 12:00:22 +0800
Subject: [PATCH 073/200] fix(migration): cert migrate error

---
 .../templates/controller-job-upgrade.yaml     |  1 +
 ...ame_procfile_type_domain_ptype_and_more.py |  5 ++---
 ..._migration_permissions_and_certificates.py | 16 ++++++----------
 .../migrations/0014_alter_certificate_app.py  | 19 +++++++++++++++++++
 4 files changed, 28 insertions(+), 13 deletions(-)
 create mode 100644 rootfs/api/migrations/0014_alter_certificate_app.py

diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index 0240f2cb7..4c09e3f8d 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -9,6 +9,7 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccount: drycc-controller
       initContainers:
       - name: drycc-controller-job-upgrade-init
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
diff --git a/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py b/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
index 730d3bd36..8214d715b 100644
--- a/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
+++ b/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
@@ -1,4 +1,4 @@
-# Generated by Django 4.2.15 on 2024-09-03 03:48
+# Generated by Django 4.2.15 on 2024-09-06 03:57
 
 import api.utils
 from django.db import migrations, models
@@ -34,8 +34,7 @@ class Migration(migrations.Migration):
         migrations.AddField(
             model_name='certificate',
             name='app',
-            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='api.app'),
-            preserve_default=False,
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.app'),
         ),
         migrations.AddField(
             model_name='release',
diff --git a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
index 7ba105043..7a2e5f81c 100644
--- a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -18,16 +18,12 @@ def migration_permission(apps, schema_editor):
 def migration_certificate(apps, schema_editor):
     for domain in Domain.objects.all():
         if domain.certificate:
-            if domain.certificate.app == None:
-                domain.certificate.app = domain.app
-                domain.certificate.save()
-            else:
-                certificate = domain.certificate
-                certificate.pk = None
-                certificate.app = domain.app
-                certificate.save()
-                domain.certificate = certificate
-                domain.save()
+            certificate = domain.certificate
+            certificate.pk = None
+            certificate.app = domain.app
+            certificate.save()
+            domain.certificate = certificate
+            domain.save()
     Certificate.objects.filter(app=None).delete()
 
 
diff --git a/rootfs/api/migrations/0014_alter_certificate_app.py b/rootfs/api/migrations/0014_alter_certificate_app.py
new file mode 100644
index 000000000..8d536fd7e
--- /dev/null
+++ b/rootfs/api/migrations/0014_alter_certificate_app.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0013_migration_permissions_and_certificates'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='certificate',
+            name='app',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.app'),
+        ),
+    ]

From 5d564ab2ef291bd9242563d5d39213e16c66aeab Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 12 Sep 2024 17:34:34 +0800
Subject: [PATCH 074/200] chore(celery): use quorum queye

---
 charts/controller/templates/_helpers.tpl |  2 +-
 rootfs/api/settings/celery.py            | 36 ++++++++----------------
 rootfs/requirements.txt                  |  2 +-
 3 files changed, 14 insertions(+), 26 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 2ff44e7d0..d8db87817 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -170,7 +170,7 @@ env:
       name: rabbitmq-creds
       key: password
 - name: "DRYCC_RABBITMQ_URL"
-  value: "amqp://$(DRYCC_RABBITMQ_USERNAME):$(DRYCC_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/drycc"
+  value: "amqp://$(DRYCC_RABBITMQ_USERNAME):$(DRYCC_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/controller"
 {{- end }}
 {{- if eq .Values.global.passportLocation "on-cluster"}}
 - name: "DRYCC_PASSPORT_URL"
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
index 9fd2f3945..78c33ed44 100644
--- a/rootfs/api/settings/celery.py
+++ b/rootfs/api/settings/celery.py
@@ -33,53 +33,41 @@ class Config(object):
     task_routes={
         'api.tasks.scale_app': {
             'queue': 'controller.high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
         'api.tasks.mount_app': {
             'queue': 'controller.high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
         'api.tasks.restart_app': {
             'queue': 'controller.high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
         'api.tasks.run_pipeline': {
             'queue': 'controller.high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
         'api.tasks.downstream_model_owner': {
             'queue': 'controller.high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
         'api.tasks.send_measurements': {
             'queue': 'controller.middle',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.middle',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.middle',
         },
     },
     task_queues=(
         Queue(
-            'controller.low',
-            exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.low',
-            queue_arguments={'x-max-priority': 16},
+            'controller.low', exchange=Exchange('controller.priority', type="direct"),
+            routing_key='controller.priority.low', queue_arguments={'x-queue-type': 'quorum'},
         ),
         Queue(
-            'controller.high',
-            exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.high',
-            queue_arguments={'x-max-priority': 64},
+            'controller.high', exchange=Exchange('controller.priority', type="direct"),
+            routing_key='controller.priority.high', queue_arguments={'x-queue-type': 'quorum'},
         ),
         Queue(
-            'controller.middle',
-            exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.middle',
-            queue_arguments={'x-max-priority': 32},
+            'controller.middle', exchange=Exchange('controller.priority', type="direct"),
+            routing_key='controller.priority.middle', queue_arguments={'x-queue-type': 'quorum'},
         ),
     ),
 )
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index dac0e9403..477391654 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -22,7 +22,7 @@ ndg-httpsclient==0.5.1
 pytz==2024.1
 requests==2.32.0
 requests-toolbelt==1.0.0
-celery==5.3.6
+celery==5.5.0b3
 hiredis==2.3.2
 django_redis==5.4.0
 dj-database-url==2.1.0

From b9c0d12852872a54de9b0952a5c3c50085ffa4a8 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sat, 14 Sep 2024 10:33:44 +0800
Subject: [PATCH 075/200] chore(filer): zero copy upload file

---
 rootfs/api/files/__init__.py |  0
 rootfs/api/files/parsers.py  | 45 ++++++++++++++++++++++++++++++++++++
 rootfs/api/views.py          |  8 +++----
 3 files changed, 49 insertions(+), 4 deletions(-)
 create mode 100644 rootfs/api/files/__init__.py
 create mode 100644 rootfs/api/files/parsers.py

diff --git a/rootfs/api/files/__init__.py b/rootfs/api/files/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/rootfs/api/files/parsers.py b/rootfs/api/files/parsers.py
new file mode 100644
index 000000000..311449463
--- /dev/null
+++ b/rootfs/api/files/parsers.py
@@ -0,0 +1,45 @@
+import contextlib
+from django.utils.http import parse_header_parameters
+from django.core.files.uploadedfile import UploadedFile
+from rest_framework.exceptions import ParseError
+from rest_framework.parsers import FileUploadParser, DataAndFiles
+
+
+class FilerFile(UploadedFile):
+
+    def __init__(self, filename, filepath, input_data, content_length):
+        self.filename = filename
+        self.filepath = filepath
+        super().__init__(input_data, name=filename, size=content_length)
+
+
+class FilerUploadParser(FileUploadParser):
+    """
+    Filer upload parser.
+    """
+    media_type = 'filer/octet-stream'
+
+    def parse(self, stream, media_type=None, parser_context=None):
+        request = parser_context['request']
+        filename = self.get_filename(stream, media_type, parser_context)
+        if not filename:
+            raise ParseError(self.errors['no_filename'])
+
+        try:
+            content_length = int(request.META.get('HTTP_CONTENT_LENGTH',
+                                                  request.META.get('CONTENT_LENGTH', 0)))
+        except (ValueError, TypeError):
+            content_length = None
+
+        file_meta = self.get_file_meta(request.META)
+        return DataAndFiles({}, {'file': FilerFile(
+            file_meta['filename'], file_meta['filepath'], stream, content_length)})
+
+    def get_file_meta(self, META):
+        """
+        Detects the uploaded file name. First searches a 'filename' url kwarg.
+        Then tries to parse Content-Disposition header.
+        """
+        with contextlib.suppress(AttributeError, KeyError, ValueError):
+            _, params = parse_header_parameters(META['HTTP_CONTENT_DISPOSITION'])
+            return params
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index e898c53ca..23629f036 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -22,7 +22,6 @@
 from rest_framework.permissions import IsAuthenticated, AllowAny
 from rest_framework.response import Response
 from rest_framework.viewsets import GenericViewSet
-from rest_framework.parsers import MultiPartParser
 from rest_framework.exceptions import PermissionDenied
 
 from api import monitor, models, permissions, serializers, viewsets, authentication
@@ -40,6 +39,7 @@
 from api import admissions, utils, filer
 from api.backend import OauthCacheManager
 from api.apps_extra.social_core.actions import do_auth, do_complete
+from api.files.parsers import FilerUploadParser
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -896,7 +896,7 @@ def path(self, request, *args, **kwargs):
 class AppFilerClientViewSet(AppResourceViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
-    parser_classes = [MultiPartParser]
+    parser_classes = [FilerUploadParser]
 
     def get_client(self):
         volume = get_object_or_404(
@@ -926,9 +926,9 @@ def retrieve(self, request, **kwargs):
         )
 
     def create(self, request, **kwargs):
-        path = request.data.get('path', '')
         client = self.get_client()
-        response = client.post(path, files=request.FILES)
+        file = request.data['file']
+        response = client.post(file.filepath, files=request.FILES)
         return Response(data=response.content, status=response.status_code)
 
     def destroy(self, request, **kwargs):

From 1eee52aa206d0388896a0be9ccd801dafe643e3e Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Sat, 14 Sep 2024 11:31:56 +0800
Subject: [PATCH 076/200] fix(controller): no cert domain do not redirect https
 (#162)

---
 rootfs/api/models/gateway.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index a411309c4..9ad67ee09 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -3,6 +3,7 @@
 from django.conf import settings
 from django.shortcuts import get_object_or_404
 from django.contrib.auth import get_user_model
+from django.db.models import Q
 
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
@@ -196,6 +197,15 @@ def hostnames(self):
         return [domain.domain for domain in self.app.domain_set.filter(
                 ptype=self.ptype)]
 
+    @property
+    def tls_force_hostnames(self):
+        tls = self.app.tls_set.latest()
+        q = Q(ptype=self.ptype)
+        if not tls.certs_auto_enabled:
+            q &= Q(certificate__isnull=False)
+        domains = self.app.domain_set.filter(q)
+        return [domain.domain for domain in domains]
+
     @property
     def default_rules(self):
         service = get_object_or_404(self.app.service_set, ptype=self.ptype)
@@ -350,7 +360,7 @@ def _https_enforced_to_k8s(self, parent_refs):
                     }
                 }]
             }],
-            "hostnames": self.hostnames,
+            "hostnames": self.tls_force_hostnames,
             "parent_refs": parent_refs,
         }
         try:

From 803da90af5f7b879ef892c03f6b3cd4a8ab4b067 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sat, 14 Sep 2024 13:36:17 +0800
Subject: [PATCH 077/200] chore(release): check deploy release ptypes

---
 rootfs/api/models/app.py   | 2 +-
 rootfs/api/models/build.py | 3 ++-
 rootfs/api/views.py        | 9 ++++++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 017b968db..52c1239e2 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -151,7 +151,7 @@ def save(self, *args, **kwargs):
     def ptypes(self):
         return list(self.structure.keys())
 
-    def check_ptypes(self, ptypes):
+    def check_ptypes(self, ptypes: set):
         """
         check available procfile types
         """
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 4d0f300b6..930989be4 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -131,7 +131,8 @@ def save(self, **kwargs):
         )
 
         # spin down any proc type removed between the last procfile and the newest one
-        if remove_procs and previous_release.build is not None:
+        if remove_procs and previous_release.build is not None and \
+                self.app.appsettings_set.latest().autodeploy:
             removed = {}
             for proc in previous_release.ptypes:
                 if proc not in self.ptypes and self.app.structure.get(proc, 0) > 0:
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 23629f036..723a499e7 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -620,10 +620,17 @@ def get_object(self, **kwargs):
     def deploy(self, request, **kwargs):
         """Deploy the latest release"""
         latest_release = self.get_app().release_set.latest()
+
         force_deploy = request.data.get("force", False)
         ptypes = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        ptypes = latest_release.app.check_ptypes(ptypes)
+        if not ptypes:
+            ptypes = latest_release.ptypes
+        else:
+            invalid_ptypes = ptypes.difference(
+                latest_release.ptypes + [d["name"] for d in self.get_app().list_deployments()])
+            if len(invalid_ptypes) != 0:
+                raise DryccException(f"process type {','.join(invalid_ptypes)} is not exists")
         latest_release.deploy(ptypes, force_deploy)
         return Response(status=status.HTTP_204_NO_CONTENT)
 

From ce20b62a139906bad1a1e4cc86e15d6138d1d48d Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 18 Sep 2024 17:35:23 +0800
Subject: [PATCH 078/200] fix(migrations): certificate migrate err

---
 .../0013_migration_permissions_and_certificates.py    | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
index 7a2e5f81c..f5d4644e0 100644
--- a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -18,10 +18,13 @@ def migration_permission(apps, schema_editor):
 def migration_certificate(apps, schema_editor):
     for domain in Domain.objects.all():
         if domain.certificate:
-            certificate = domain.certificate
-            certificate.pk = None
-            certificate.app = domain.app
-            certificate.save()
+            certificate = Certificate.objects.filter(
+                app=domain.app, name=domain.certificate.name).first()
+            if not certificate:
+                certificate = domain.certificate
+                certificate.pk = None
+                certificate.app = domain.app
+                certificate.save()
             domain.certificate = certificate
             domain.save()
     Certificate.objects.filter(app=None).delete()

From ad13e300db86dfbaa604f200d18ab7dde4fb6e79 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 19 Sep 2024 09:06:02 +0800
Subject: [PATCH 079/200] feat(release): add query release version

---
 rootfs/api/tests/test_config.py | 41 +++++++++++++++++++++++++++++++++
 rootfs/api/views.py             | 11 +++++----
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 996a002d3..884e1c45a 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -587,3 +587,44 @@ def test_config_set_typed_values(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(response.data['typed_values'], {}, response.data)
+
+    def test_config_version(self, mock_requests):
+        """
+        Test that config sets on the same key function properly
+        """
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+
+        # set an initial config value
+        body = {'values': {'PORT': '5000'}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertIn('PORT', response.data['values'])
+        url = f"/v2/apps/{app_id}/config"
+
+        # set config NAME
+        body = {'values': {'NAME': 'drycc'}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertIn('NAME', response.data['values'])
+
+        # set config WEBSITE
+        body = {'values': {'WEBSITE': 'www.drycc.cc'}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertIn('WEBSITE', response.data['values'])
+
+        url = f"/v2/apps/{app_id}/config/?version=v2"
+        response = self.client.get(url)
+        self.assertEqual(response.data['values'], {'PORT': '5000'}, response.data)
+
+        url = f"/v2/apps/{app_id}/config/?version=v3"
+        response = self.client.get(url)
+        self.assertEqual(
+            response.data['values'], {'PORT': '5000', 'NAME': 'drycc'}, response.data)
+
+        url = f"/v2/apps/{app_id}/config/?version=v4"
+        response = self.client.get(url)
+        self.assertEqual(
+            response.data['values'],
+            {'PORT': '5000', 'NAME': 'drycc', 'WEBSITE': 'www.drycc.cc'}, response.data)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 723a499e7..be50f7174 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -272,10 +272,13 @@ class ReleasableViewSet(AppResourceViewSet):
     """A viewset for application resources which affect the release cycle."""
     def get_object(self):
         """Retrieve the object based on the latest release's value"""
-        return getattr(
-            self.get_app().release_set.filter(failed=False).latest(),
-            self.model.__name__.lower()
-        )
+        version = self.request.query_params.get('version', '').lower().strip('v')
+        if re.search("^[0-9]+", version):
+            release = get_object_or_404(
+                models.release.Release, app=self.get_app(), version=version)
+        else:
+            release = self.get_app().release_set.filter(failed=False).latest()
+        return getattr(release, self.model.__name__.lower())
 
 
 class AppViewSet(BaseDryccViewSet):

From 53968f96169a6fd4517690d97279d62b7c0893f8 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 19 Sep 2024 14:43:00 +0800
Subject: [PATCH 080/200] feat(build): add get build api

---
 rootfs/api/tests/test_app.py          | 16 +++---
 rootfs/api/tests/test_build.py        | 80 +++++++++++++--------------
 rootfs/api/tests/test_config.py       |  6 +-
 rootfs/api/tests/test_event.py        |  2 +-
 rootfs/api/tests/test_healthchecks.py |  2 +-
 rootfs/api/tests/test_hooks.py        | 10 ++--
 rootfs/api/tests/test_pods.py         | 20 +++----
 rootfs/api/tests/test_ptypes.py       |  4 +-
 rootfs/api/tests/test_release.py      | 26 ++++-----
 rootfs/api/tests/test_volume.py       |  2 +-
 rootfs/api/urls.py                    |  9 +--
 rootfs/api/views.py                   |  5 +-
 12 files changed, 88 insertions(+), 94 deletions(-)

diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index c92f4d215..922262891 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -247,7 +247,7 @@ def test_run(self, mock_requests):
 
         # create build
         body = {'image': 'autotest/example', 'stack': 'container'}
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -271,7 +271,7 @@ def test_run_failure(self, mock_requests):
 
         # create build
         body = {'image': 'autotest/example', 'stack': 'container'}
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -453,7 +453,7 @@ def test_app_verify_application_health_success(self, mock_requests):
         mr = mock_requests.register_uri('GET', hostname, responses)
 
         # deploy app to get verification
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -488,7 +488,7 @@ def test_app_verify_application_health_failure_404(self, mock_requests):
             app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
         mr = mock_requests.register_uri('GET', hostname, responses)
         # deploy app to get verification
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -513,7 +513,7 @@ def _raise_exception(request, ctx):
         mr = mock_requests.register_uri('GET', hostname, text=_raise_exception)
 
         # deploy app to get verification
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -589,7 +589,7 @@ def test_build_env_vars(self, mock_requests):
         with self.assertRaises(DryccException):
             app._build_env_vars(app.release_set.latest(), PTYPE_WEB)
         data = {'image': 'autotest/example', 'stack': 'heroku-18'}
-        url = f"/v2/apps/{app.id}/builds"
+        url = f"/v2/apps/{app.id}/build"
         response = self.client.post(url, data)
         self.assertEqual(response.status_code, 201, response.data)
         time_created = app.release_set.latest().created
@@ -624,7 +624,7 @@ def test_gather_app_settings(self, mock_requests):
         app = App.objects.create(owner=self.user)
         app.save()
         data = {'image': 'autotest/example', 'stack': 'container'}
-        url = f"/v2/apps/{app.id}/builds"
+        url = f"/v2/apps/{app.id}/build"
         response = self.client.post(url, data)
         self.assertEqual(response.status_code, 201, response.data)
         Config.objects.create(
@@ -659,7 +659,7 @@ def test_app_name_bad_regex(self, mock_requests):
         self.assertEqual(response.data['results'][0]['id'], app_id, response.data)
 
         # deploy to an app that doesn't exist should fail with 404
-        url = "/v2/apps/{}/builds".format('={}'.format(app_id))
+        url = "/v2/apps/{}/build".format('={}'.format(app_id))
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 404, response)
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 7993bacb9..57be0485d 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -44,28 +44,26 @@ def test_build(self, mock_requests):
         app_id = self.create_app()
 
         # check to see that no initial build was created
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 0)
 
         # post a new build
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        build_id = str(response.data['uuid'])
         build1 = response.data
         self.assertEqual(response.data['image'], body['image'])
 
         # read the build
-        url = f"/v2/apps/{app_id}/builds/{build_id}"
+        url = f"/v2/apps/{app_id}/build"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         build2 = response.data
         self.assertEqual(build1, build2)
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -86,7 +84,7 @@ def test_response_data(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
 
@@ -108,7 +106,7 @@ def test_build_default_containers(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -121,7 +119,7 @@ def test_build_default_containers(self, mock_requests):
         # post an image as a build with a procfile
         app_id = self.create_app()
         # post an image as a build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -140,7 +138,7 @@ def test_build_default_containers(self, mock_requests):
         # start with a new app
         app_id = self.create_app()
         # post a new build with procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -158,7 +156,7 @@ def test_build_default_containers(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build with procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -179,7 +177,7 @@ def test_build_default_containers(self, mock_requests):
         app_id = self.create_app()
         # post a new build with procfile
 
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -201,7 +199,7 @@ def test_build_default_containers(self, mock_requests):
         app_id = self.create_app()
         # post a new build with procfile and no routable type
 
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -225,7 +223,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build with procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -256,7 +254,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -290,7 +288,7 @@ def test_build_no_remove_process(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build with procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -316,7 +314,7 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -362,7 +360,7 @@ def test_build_forgotten_procfile_reject(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build with procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -388,7 +386,7 @@ def test_build_forgotten_procfile_reject(self, mock_requests):
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 409, response.data)
@@ -398,7 +396,7 @@ def test_build_str(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -419,7 +417,7 @@ def test_admin_can_create_builds_on_other_apps(self, mock_requests):
 
         # post a new build as admin
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -440,7 +438,7 @@ def test_unauthorized_user_cannot_modify_build(self, mock_requests):
         unauthorized_user = User.objects.get(username='autotest2')
         unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
-        url = '/v2/apps/{}/builds'.format(app_id)
+        url = '/v2/apps/{}/build'.format(app_id)
         body = {'image': 'foo'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
@@ -453,7 +451,7 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -478,7 +476,7 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         self.assertEqual(response.status_code, 204, response.data)
 
         # post another build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -500,7 +498,7 @@ def test_build_image_in_registry(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         image = '127.0.0.1:5000/autotest/example'
         body = {'image': image, 'stack': 'container'}
         response = self.client.post(url, body)
@@ -511,7 +509,7 @@ def test_build_image_in_registry(self, mock_requests):
         self.assertEqual(release.get_deploy_image(PTYPE_WEB), image)
 
         # post an image as a build using registry hostname + port
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         image = '127.0.0.1:5000/autotest/example'
         body = {'image': image, 'stack': 'container'}
         response = self.client.post(url, body)
@@ -526,7 +524,7 @@ def test_build_image_in_registry_with_auth(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         image = 'autotest/example'
         response = self.client.post(url, {'image': image, 'stack': 'container'})
         self.assertEqual(response.status_code, 201, response.data)
@@ -548,7 +546,7 @@ def test_build_image_in_registry_with_auth_no_port(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         image = 'autotest/example'
         response = self.client.post(url, {'image': image, 'stack': 'container'})
         self.assertEqual(response.status_code, 201, response.data)
@@ -566,7 +564,7 @@ def test_release_create_failure(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app to get a build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -575,7 +573,7 @@ def test_release_create_failure(self, mock_requests):
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
 
-            url = f"/v2/apps/{app_id}/builds"
+            url = f"/v2/apps/{app_id}/build"
             body = {'image': 'autotest/example'}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
@@ -589,7 +587,7 @@ def test_build_deploy_kube_failure(self, mock_requests):
         with mock.patch('scheduler.KubeHTTPClient.deploy') as mock_deploy:
             mock_deploy.side_effect = KubeException('Boom!')
 
-            url = f"/v2/apps/{app_id}/builds"
+            url = f"/v2/apps/{app_id}/build"
             body = {'image': 'autotest/example', 'stack': 'container'}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
@@ -601,7 +599,7 @@ def test_build_failures(self, mock_requests):
         app = App.objects.get(id=app_id)
 
         # deploy app to get a build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -612,7 +610,7 @@ def test_build_failures(self, mock_requests):
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
 
-            url = f"/v2/apps/{app_id}/builds"
+            url = f"/v2/apps/{app_id}/build"
             body = {'image': 'autotest/example', 'stack': 'container'}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
@@ -633,7 +631,7 @@ def test_build_validate_procfile(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app with incorrect proctype
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -646,7 +644,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -659,7 +657,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -672,7 +670,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -684,7 +682,7 @@ def test_build_validate_procfile(self, mock_requests):
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -697,7 +695,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
         # deploy app with empty command
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -710,7 +708,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -725,7 +723,7 @@ def test_build_validate_procfile(self, mock_requests):
 
     def test_dryccfile_ok(self, mock_requests):
         app_id = self.create_app()
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         default_image = "autotest/example"
         run_image = "127.0.0.1:7070/myapp/run:git-123fsa1"
         web_image = "127.0.0.1:7070/myapp/web:git-123fsa1"
@@ -797,7 +795,7 @@ def test_dryccfile_format(self, mock_requests):
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             app_id = self.create_app()
-            url = f"/v2/apps/{app_id}/builds"
+            url = f"/v2/apps/{app_id}/build"
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
             body['dryccfile']['deploy'] = {
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 884e1c45a..42834edd7 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -264,7 +264,7 @@ def test_config_deploy_failure(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app to get a build
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -363,7 +363,7 @@ def test_config_failures(self, mock_requests):
         app = App.objects.get(id=app_id)
 
         # deploy app to get a build
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -390,7 +390,7 @@ def test_config_failures(self, mock_requests):
             self.assertEqual(app.release_set.filter(failed=False).latest().version, 3)
 
         # create a build to see that the new release is created with the last successful config
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_event.py b/rootfs/api/tests/test_event.py
index 9e323a4d2..cf44d61b2 100644
--- a/rootfs/api/tests/test_event.py
+++ b/rootfs/api/tests/test_event.py
@@ -37,7 +37,7 @@ def tearDown(self):
 
     def build_deploy(self, app_id):
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        url = "/v2/apps/{app_id}/build".format(app_id=app_id)
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
diff --git a/rootfs/api/tests/test_healthchecks.py b/rootfs/api/tests/test_healthchecks.py
index 06fc730e5..b4e9db8a5 100644
--- a/rootfs/api/tests/test_healthchecks.py
+++ b/rootfs/api/tests/test_healthchecks.py
@@ -91,7 +91,7 @@ def test_config_healthchecks(self, mock_requests):
 
         # post a new build
         response = self.client.post(
-            f"/v2/apps/{app_id}/builds",
+            f"/v2/apps/{app_id}/build",
             {'image': 'quay.io/autotest/example', 'stack': 'container'}
         )
         self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index fc85ed369..0056d4a20 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -225,11 +225,10 @@ def test_build_hook_procfile(self, mock_requests):
         self.assertIn('version', response.data['release'])
 
         # make sure build fields were populated
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertIn('results', response.data)
-        build = response.data['results'][0]
+        build = response.data
         self.assertEqual(build['sha'], SHA)
         self.assertEqual(build['procfile'], PROCFILE)
 
@@ -266,11 +265,10 @@ def test_build_hook_dockerfile(self, mock_requests):
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
         # make sure build fields were populated
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertIn('results', response.data)
-        build = response.data['results'][0]
+        build = response.data
         self.assertEqual(build['sha'], SHA)
         self.assertEqual(build['dockerfile'], DOCKERFILE)
         # test default container
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index f0581b4ad..f9ebf32e4 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -46,7 +46,7 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -126,7 +126,7 @@ def test_container_api(self, mock_requests):
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'container',
@@ -190,7 +190,7 @@ def test_release(self, mock_requests):
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -215,7 +215,7 @@ def test_release(self, mock_requests):
         self.assertPodContains(response.data['results'], app_id, "web", "v2")
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         # a web proctype must exist on the second build or else the container will be removed
         body = {
             'image': 'autotest/example',
@@ -297,7 +297,7 @@ def test_container_str(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -335,7 +335,7 @@ def test_pod_command_format(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -381,7 +381,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -453,7 +453,7 @@ def test_admin_can_manage_other_pods(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -700,7 +700,7 @@ def test_scale_with_unauthorized_user_returns_403(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -725,7 +725,7 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        build_url = f"/v2/apps/{app_id}/builds"
+        build_url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
index fcda2092c..64749c3fe 100644
--- a/rootfs/api/tests/test_ptypes.py
+++ b/rootfs/api/tests/test_ptypes.py
@@ -37,7 +37,7 @@ def tearDown(self):
 
     def build_deploy(self, app_id):
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        url = "/v2/apps/{app_id}/build".format(app_id=app_id)
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -76,7 +76,7 @@ def test_restart_ptypes(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        build_url = f"/v2/apps/{app_id}/builds"
+        build_url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index ea859dde7..2924a29b4 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -72,7 +72,7 @@ def test_release(self, mock_requests):
         self.assertEqual(release1['build'], release2['build'])
         self.assertEqual(release2['version'], 2)
         # check that updating the build rolls a new release
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -107,7 +107,7 @@ def test_release(self, mock_requests):
 
     def test_conditions(self, mock_requests):
         app_id = self.create_app()
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -125,7 +125,7 @@ def test_conditions(self, mock_requests):
 
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
-            url = f"/v2/apps/{app_id}/builds"
+            url = f"/v2/apps/{app_id}/build"
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
             url = f'/v2/apps/{app_id}/releases/'
@@ -134,7 +134,7 @@ def test_conditions(self, mock_requests):
 
     def test_get_image(self, mock_requests):
         app_id = self.create_app()
-        url = f"/v2/apps/{app_id}/builds"
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': '127.0.0.1:5555/autotest/example:git-fadf1231',
             'stack': 'heroku-18',
@@ -242,7 +242,7 @@ def test_release_deploy(self, mock_requests):
             self.assertEqual(response.status_code, 400)
 
         # post a new build
-        build_url = f"/v2/apps/{app_id}/builds"
+        build_url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -270,7 +270,7 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(response.data, {'detail': 'version cannot be below 0'})
         self.assertEqual(response.get('content-type'), 'application/json')
         # update the build to roll a new release
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -280,7 +280,7 @@ def test_release_rollback(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # create another release with a different build
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'autotest/example:latest', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -373,7 +373,7 @@ def test_unauthorized_user_cannot_modify_release(self, mock_requests):
         app_id = self.create_app()
 
         # push a new build
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'test', 'stack': 'container'}
         response = self.client.post(url, body)
 
@@ -397,7 +397,7 @@ def test_release_rollback_failure(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app to get a build
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -496,7 +496,7 @@ def test_release_get_port(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
 
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'sha': '123456', 'image': 'autotest/example', 'stack': 'heroku-18'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -506,7 +506,7 @@ def test_release_get_port(self, mock_requests):
         self.assertEqual(release.get_port('web'), 5000)
 
         # switch to a dockerfile app or else it'll automatically default to 5000
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -559,7 +559,7 @@ def test_deploy_hooks_logged(self, mock_requests, mock_logger):
 
         # override DRYCC_DEPLOY_HOOK_URLS again, ensuring that the new deploy hooks get the same
         # treatment
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         with self.settings(DRYCC_DEPLOY_HOOK_URLS=['http://drycc.ninja', 'http://cat.dog']):
             mr_ninja = mock_requests.post(f"http://drycc.ninja?app={app_id}&user={self.user.username}&sha=123456&release=v2&release_summary={self.user.username}+deployed+123456")  # noqa
             mr_catdog = mock_requests.post(f"http://cat.dog?app={app_id}&user={self.user.username}&sha=123456&release=v2&release_summary={self.user.username}+deployed+123456")  # noqa
@@ -645,7 +645,7 @@ def test_release_external_registry(self, mock_requests):
         self.assertEqual(config_response.status_code, 201, config_response.data)
 
         app = App.objects.get(id=app_id)
-        url = f'/v2/apps/{app_id}/builds'
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'test/autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index 1f97239dd..e3d180593 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -228,7 +228,7 @@ def test_volume_unmount(self, mock_requests):
 
     def build_deploy(self, app_id):
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        url = "/v2/apps/{app_id}/build".format(app_id=app_id)
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 805d42010..dade13b81 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -27,15 +27,12 @@
         r'^limits/plans/(?P<id>[-.\w]+)/?$',
         views.LimitPlanViewSet.as_view({'get': 'retrieve'})),
     # application release components
+    re_path(
+        r"^apps/(?P<id>{})/build/?$".format(settings.APP_URL_REGEX),
+        views.BuildViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
     re_path(
         r"^apps/(?P<id>{})/config/?$".format(settings.APP_URL_REGEX),
         views.ConfigViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
-    re_path(
-        r"^apps/(?P<id>{})/builds/(?P<uuid>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
-        views.BuildViewSet.as_view({'get': 'retrieve'})),
-    re_path(
-        r"^apps/(?P<id>{})/builds/?$".format(settings.APP_URL_REGEX),
-        views.BuildViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
         r"^apps/(?P<id>{})/releases/v(?P<version>[0-9]+)/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'get': 'retrieve'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index be50f7174..0e0c7a244 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -270,12 +270,13 @@ def create(self, request, **kwargs):
 
 class ReleasableViewSet(AppResourceViewSet):
     """A viewset for application resources which affect the release cycle."""
+
     def get_object(self):
         """Retrieve the object based on the latest release's value"""
         version = self.request.query_params.get('version', '').lower().strip('v')
-        if re.search("^[0-9]+", version):
+        if re.search("^[0-9]+$", version):
             release = get_object_or_404(
-                models.release.Release, app=self.get_app(), version=version)
+                models.release.Release, app=self.get_app(), version=int(version))
         else:
             release = self.get_app().release_set.filter(failed=False).latest()
         return getattr(release, self.model.__name__.lower())

From 6914ad47c2942a1a6408b94cfe75031be65e19e6 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 20 Sep 2024 10:51:13 +0800
Subject: [PATCH 081/200] chore(filer): add pod security context

---
 rootfs/api/filer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
index 84a1eab8e..daff5a475 100644
--- a/rootfs/api/filer.py
+++ b/rootfs/api/filer.py
@@ -54,6 +54,7 @@ def get_server(self):
             "app_type": "filer", "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
             "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
             "restart_policy": "Never", "image_pull_policy": settings.DRYCC_FILER_IMAGE_PULL_POLICY,
+            "pod_security_context": {"fsGroup": 1001, "runAsGroup": 1001, "runAsUser": 1001},
         })
         address = self.scheduler.pod.get(self.app_id, pod_name).json()["status"]["podIP"]
         return {"address": address, "username": username, "password": password}

From f18c36eed0589bdd0a89da3edb563e2e34574363 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 20 Sep 2024 13:44:06 +0800
Subject: [PATCH 082/200] chore(controller): volume not related release (#164)

---
 rootfs/api/views.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 0e0c7a244..a20330dbd 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -858,7 +858,7 @@ def destroy(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class AppVolumesViewSet(ReleasableViewSet):
+class AppVolumesViewSet(AppResourceViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
     serializer_class = serializers.VolumeSerializer

From 849dd602a70e8d905fd5a32b9908a5e6bc492d9e Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 20 Sep 2024 21:20:51 +0800
Subject: [PATCH 083/200] chore(filer): url quote content disposition

---
 rootfs/api/files/parsers.py | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/rootfs/api/files/parsers.py b/rootfs/api/files/parsers.py
index 311449463..867b943c3 100644
--- a/rootfs/api/files/parsers.py
+++ b/rootfs/api/files/parsers.py
@@ -19,27 +19,40 @@ class FilerUploadParser(FileUploadParser):
     """
     media_type = 'filer/octet-stream'
 
+    errors = {
+        "no_filepath": (
+            "Missing filepath. Request should include a Content-Disposition "
+            "header with a filepath parameter."
+        ),
+    }
+    errors.update(FileUploadParser.errors)
+
     def parse(self, stream, media_type=None, parser_context=None):
         request = parser_context['request']
         filename = self.get_filename(stream, media_type, parser_context)
         if not filename:
             raise ParseError(self.errors['no_filename'])
-
+        filepath = self.get_filepath(stream, media_type, parser_context)
+        if not filepath:
+            raise ParseError(self.errors['no_filepath'])
         try:
             content_length = int(request.META.get('HTTP_CONTENT_LENGTH',
                                                   request.META.get('CONTENT_LENGTH', 0)))
         except (ValueError, TypeError):
             content_length = None
+        return DataAndFiles({}, {'file': FilerFile(filename, filepath, stream, content_length)})
 
-        file_meta = self.get_file_meta(request.META)
-        return DataAndFiles({}, {'file': FilerFile(
-            file_meta['filename'], file_meta['filepath'], stream, content_length)})
-
-    def get_file_meta(self, META):
+    def get_filepath(self, stream, media_type, parser_context):
         """
-        Detects the uploaded file name. First searches a 'filename' url kwarg.
+        Detects the uploaded file name. First searches a 'filepath' url kwarg.
         Then tries to parse Content-Disposition header.
         """
+        with contextlib.suppress(KeyError):
+            return parser_context['kwargs']['filepath']
+
         with contextlib.suppress(AttributeError, KeyError, ValueError):
-            _, params = parse_header_parameters(META['HTTP_CONTENT_DISPOSITION'])
-            return params
+            meta = parser_context['request'].META
+            _, params = parse_header_parameters(meta['HTTP_CONTENT_DISPOSITION'])
+            if 'filepath*' in params:
+                return params['filepath*']
+            return params['filepath']

From 117b407c2ea2936d183410ba16f47be43981208f Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 25 Sep 2024 08:54:40 +0800
Subject: [PATCH 084/200] fix(controller): app_settings boolean field cannot be
 set correctly (#166)

---
 ...5_alter_appsettings_autodeploy_and_more.py | 28 ++++++++++
 rootfs/api/models/app.py                      |  3 +-
 rootfs/api/models/appsettings.py              |  8 ++-
 rootfs/api/tests/test_app_settings.py         | 56 +++++++++++++------
 4 files changed, 74 insertions(+), 21 deletions(-)
 create mode 100644 rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py

diff --git a/rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py b/rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py
new file mode 100644
index 000000000..f3786e0a8
--- /dev/null
+++ b/rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.15 on 2024-09-24 09:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0014_alter_certificate_app'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='appsettings',
+            name='autodeploy',
+            field=models.BooleanField(default=None),
+        ),
+        migrations.AlterField(
+            model_name='appsettings',
+            name='autorollback',
+            field=models.BooleanField(default=None),
+        ),
+        migrations.AlterField(
+            model_name='appsettings',
+            name='routable',
+            field=models.BooleanField(default=None),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 52c1239e2..464eac2b5 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -204,7 +204,8 @@ def create(self, *args, **kwargs):  # noqa
         try:
             self.appsettings_set.latest()
         except AppSettings.DoesNotExist:
-            AppSettings.objects.create(owner=self.owner, app=self)
+            AppSettings.objects.create(
+                owner=self.owner, app=self, routable=True, autodeploy=True, autorollback=True)
         try:
             self.tls_set.latest()
         except TLS.DoesNotExist:
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index 61282ad81..46d00a035 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -19,9 +19,9 @@ class AppSettings(UuidAuditedModel):
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    routable = models.BooleanField(default=True)
-    autodeploy = models.BooleanField(default=True)
-    autorollback = models.BooleanField(default=True)
+    routable = models.BooleanField(default=None)
+    autodeploy = models.BooleanField(default=None)
+    autorollback = models.BooleanField(default=None)
     autoscale = models.JSONField(default=dict, blank=True)
     label = models.JSONField(default=dict, blank=True)
 
@@ -70,6 +70,8 @@ def _update_field(self, field, previous_settings):
         # if nothing changed copy the settings from previous
         if new is None and old is not None:
             setattr(self, field, old)
+        elif new is None and isinstance(self._meta.get_field(field), models.BooleanField):
+            setattr(self, field, True)
         elif old != new:
             self.summary += ["{} changed {} from {} to {}".format(self.owner, field, old, new)]
 
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
index 7f4b621bf..92cb3a9a2 100644
--- a/rootfs/api/tests/test_app_settings.py
+++ b/rootfs/api/tests/test_app_settings.py
@@ -24,7 +24,7 @@ def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
-    def test_settings_routable(self, mock_requests):
+    def test_settings_bool(self, mock_requests):
         """
         Create an application with the routable flag turned on or off
         """
@@ -32,6 +32,8 @@ def test_settings_routable(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
         self.assertTrue(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
         # Set routable to false
         response = self.client.post(
             f'/v2/apps/{app.id}/settings',
@@ -39,39 +41,59 @@ def test_settings_routable(self, mock_requests):
         )
         self.assertEqual(response.status_code, 201, response.data)
         self.assertFalse(app.appsettings_set.latest().routable)
-
-    def test_settings_autodeploy(self, mock_requests):
-        """
-        Create an application with the autorollback flag turned on or off
-        """
-        # create app, expecting autodeploy to be true
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
         self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
         # Set autodeploy to false
         response = self.client.post(
             f'/v2/apps/{app.id}/settings',
             {'autodeploy': False}
         )
         self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
         self.assertFalse(app.appsettings_set.latest().autodeploy)
-
-    def test_settings_autorollback(self, mock_requests):
-        """
-        Create an application with the autorollback flag turned on or off
-        """
-        # create app, expecting autorollback to be true
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
         self.assertTrue(app.appsettings_set.latest().autorollback)
+
         # Set autorollback to false
         response = self.client.post(
             f'/v2/apps/{app.id}/settings',
             {'autorollback': False}
         )
         self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
         self.assertFalse(app.appsettings_set.latest().autorollback)
 
+        # Set autorollback to true
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autorollback': True}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
+        # Set autodeploy to true
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autodeploy': True}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
+        # Set routable to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'routable': True}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertTrue(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
     def test_autoscale(self, mock_requests):
         """
         Test that autoscale can be applied

From 38570540eba99f1dcb50565a41c8e2fb12402822 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 9 Oct 2024 09:08:00 +0800
Subject: [PATCH 085/200] chore(deps): bump django from 4.2.15 to 4.2.16 in
 /rootfs (#167)

Bumps [django](https://github.com/django/django) from 4.2.15 to 4.2.16.
- [Commits](https://github.com/django/django/compare/4.2.15...4.2.16)

---
updated-dependencies:
- dependency-name: django
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 477391654..942de01e7 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.15
+django==4.2.16
 channels==4.1.0
 aiohttp==3.10.2
 django-cors-headers==4.3.1

From e25d7ce6573860050408bd84de24bf339e20d84d Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sat, 12 Oct 2024 09:16:38 +0800
Subject: [PATCH 086/200] chore(service): add name field

---
 rootfs/api/models/service.py | 48 +++++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 20 deletions(-)

diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index 9ad5219fb..d2958e8df 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -41,16 +41,36 @@ class Meta:
         ordering = ['-created']
 
     def __str__(self):
-        return self._svc_name()
+        return self.name
+
+    @property
+    def name(self):
+        if self.ptype == 'web':
+            svc_name = self.app.id
+        else:
+            svc_name = "{}-{}".format(self.app.id, self.ptype)
+        return svc_name
 
     @property
     def domain(self):
         return "{}.{}.svc.{}".format(
-            self._svc_name(), self._namespace(), settings.KUBERNETES_CLUSTER_DOMAIN
+            self.name, self.namespace, settings.KUBERNETES_CLUSTER_DOMAIN
         )
 
+    @property
+    def namespace(self):
+        return self.app.id
+
+    @classmethod
+    def get(cls, app, name):
+        for service in cls.objects.filter(app=app):
+            if service.name == name:
+                return service
+        raise cls.DoesNotExist()
+
     def as_dict(self):
         return {
+            "name": self.name,
             "domain": self.domain,
             "ports": self.ports,
             "ptype": self.ptype,
@@ -101,7 +121,7 @@ def save(self, *args, **kwargs):
         return service
 
     def delete(self, *args, **kwargs):
-        self._delete_k8s_svc(self._svc_name())
+        self._delete_k8s_svc(self.name)
         # Delete from DB
         return super(Service, self).delete(*args, **kwargs)
 
@@ -116,30 +136,18 @@ def log(self, message, level=logging.INFO):
         """
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
-    def _namespace(self):
-        return self.app.id
-
-    def _svc_name(self):
-        if self.ptype == 'web':
-            svc_name = self.app.id
-        else:
-            svc_name = "{}-{}".format(self.app.id, self.ptype)
-        return svc_name
-
     def refresh_k8s_svc(self):
-        svc_name = self._svc_name()
-        namespace = self._namespace()
-        self.log('creating service: {}'.format(svc_name), level=logging.DEBUG)
+        self.log('creating service: {}'.format(self.name), level=logging.DEBUG)
         try:
             try:
-                data = self.scheduler().svc.get(namespace, svc_name).json()
-                self.scheduler().svc.patch(namespace, svc_name, **{
+                data = self.scheduler().svc.get(self.namespace, self.name).json()
+                self.scheduler().svc.patch(self.namespace, self.name, **{
                     "ports": self.ports,
                     "version": data["metadata"]["resourceVersion"],
                     "ptype": self.ptype,
                 })
             except KubeException:
-                self.scheduler().svc.create(namespace, svc_name, **{
+                self.scheduler().svc.create(self.namespace, self.name, **{
                     "ports": self.ports,
                     "ptype": self.ptype,
                 })
@@ -148,4 +156,4 @@ def refresh_k8s_svc(self):
 
     def _delete_k8s_svc(self, svc_name):
         self.log('deleting Service: {}'.format(svc_name), level=logging.DEBUG)
-        self.scheduler().svc.delete(self._namespace(), svc_name, ignore_exception=True)
+        self.scheduler().svc.delete(self.namespace, svc_name, ignore_exception=True)

From af5e4b88cfb2d42c15249c7609a42af4caf4c9d2 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sat, 12 Oct 2024 14:40:45 +0800
Subject: [PATCH 087/200] feat(route): support multi backend

---
 rootfs/api/models/app.py           |  4 +-
 rootfs/api/models/gateway.py       | 87 ++++++++++++++++++------------
 rootfs/api/serializers/__init__.py | 45 ++++++----------
 rootfs/api/signals.py              |  6 ++-
 rootfs/api/tests/test_gateway.py   | 16 ++++--
 rootfs/api/views.py                | 37 ++-----------
 6 files changed, 94 insertions(+), 101 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 464eac2b5..251ad77b1 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -938,8 +938,8 @@ def _create_default_ingress(self, target_port):
                 route.save()
         except Route.DoesNotExist:
             route = Route(app=self, owner=self.owner, kind="HTTPRoute", name=self.id,
-                          port=DEFAULT_HTTP_PORT, ptype=service.ptype)
-            route.rules = route.default_rules
+                          rules=[{"backendRefs": [{"kind": "Service", "name": service.name,
+                                                   "port": DEFAULT_HTTP_PORT, "weight": 100}]}])
             attached, msg = route.attach(gateway.name, DEFAULT_HTTP_PORT)
             if not attached:
                 raise DryccException(msg)
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 9ad67ee09..904c4abe9 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -1,14 +1,15 @@
 import logging
+import threading
 from django.db import models
 from django.conf import settings
-from django.shortcuts import get_object_or_404
 from django.contrib.auth import get_user_model
 from django.db.models import Q
+from django.http import Http404
 
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 
-from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, PTYPE_MAX_LENGTH
+from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -167,6 +168,7 @@ class Meta:
 
 
 class Route(AuditedModel):
+    CACHE = threading.local()
     PROTOCOLS_CHOICES = {
         "TLSRoute": ("TCP", ),
         "TCPRoute": ("TCP", ),
@@ -180,11 +182,21 @@ class Route(AuditedModel):
     kind = models.CharField(max_length=15, choices=[
         (key, '/'.join(value)) for key, value in PROTOCOLS_CHOICES.items()])
     name = models.CharField(max_length=63, db_index=True)
-    port = models.PositiveIntegerField()
     rules = models.JSONField(default=list)
     routable = models.BooleanField(default=True)
     parent_refs = models.JSONField(default=list)
-    ptype = models.CharField(max_length=PTYPE_MAX_LENGTH)
+
+    @property
+    def services(self):
+        key = f"{self.app.id}_{self.name}"
+        if not hasattr(self.CACHE, key):
+            service_names = set()
+            for rule in self.rules:
+                for backend in rule['backendRefs']:
+                    service_names.add(backend['name'])
+            setattr(self.CACHE, key,
+                    [s for s in self.app.service_set.all() if s.name in service_names])
+        return getattr(self.CACHE, key)
 
     @property
     def protocols(self):
@@ -195,30 +207,47 @@ def protocols(self):
     @property
     def hostnames(self):
         return [domain.domain for domain in self.app.domain_set.filter(
-                ptype=self.ptype)]
+                ptype__in=[s.ptype for s in self.services])]
+
+    @property
+    def cleaned_rules(self):
+        services, rules = self.services, []
+        for rule in self.rules:
+            backend_refs = []
+            for backend_ref in rule["backendRefs"]:
+                for service in services:
+                    ports = [item["port"] for item in service.ports]
+                    if backend_ref["port"] in ports and backend_ref["name"] == service.name:
+                        backend_refs.append(backend_ref)
+            if backend_refs:
+                rule['backendRefs'] = backend_refs
+                rules.append(rule)
+        return rules
 
     @property
     def tls_force_hostnames(self):
         tls = self.app.tls_set.latest()
-        q = Q(ptype=self.ptype)
+        q = Q(ptype__int=[s.ptype for s in self.services])
         if not tls.certs_auto_enabled:
             q &= Q(certificate__isnull=False)
         domains = self.app.domain_set.filter(q)
         return [domain.domain for domain in domains]
 
-    @property
-    def default_rules(self):
-        service = get_object_or_404(self.app.service_set, ptype=self.ptype)
-        backend_refs = []
-        for item in service.ports:
-            if item["port"] == self.port:
-                backend_refs.append({
-                    "kind": "Service",
-                    "name": str(service),
-                    "port": item["port"],
-                    "weight": 100,
-                })
-        return [{"backendRefs": backend_refs}]
+    def check_rules(self, rules):
+        for rule in rules:
+            for backend_ref in rule["backendRefs"]:
+                kind = backend_ref.get("kind", "Service")
+                if kind != "Service":
+                    return False, {"detail": "BackendRef only supports service kind"}
+                has_service = False
+                for service in self.services:
+                    ports = [item["port"] for item in service.ports]
+                    if backend_ref["port"] in ports and backend_ref["name"] == service.name:
+                        has_service = True
+                        break
+                if not has_service:
+                    return False, {"detail": f"service {backend_ref['name']} not exists"}
+        return True, ""
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this service.
@@ -231,17 +260,6 @@ def log(self, message, level=logging.INFO):
         """
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
-    def check_rules(self):
-        service = self.app.service_set.filter(
-            ptype=self.ptype).first()
-        ports = [item["port"] for item in service.ports]
-        for rule in self.rules:
-            for backend_ref in rule["backendRefs"]:
-                port = backend_ref["port"]
-                if port not in ports or backend_ref["name"] != str(service):
-                    return False, {"detail": "backendRefs associated with incorrect service"}
-        return True, ""
-
     def refresh_to_k8s(self):
         if self.routable:
             parent_refs, http_parent_refs = self._get_all_parent_refs()
@@ -290,9 +308,12 @@ def detach(self, gateway_name, port):
 
     def save(self, *args, **kwargs):
         self.change_default_tls()
-        ok, msg = self.check_rules()
-        if not ok:
-            raise ValueError(msg)
+        cleaned_rules = self.cleaned_rules
+        if not cleaned_rules:
+            msg = f"route {self.name} no available backend"
+            self.log(msg, level=logging.ERROR)
+            raise Http404(msg)
+        self.rules = cleaned_rules
         super().save(*args, **kwargs)
         self.refresh_to_k8s()
 
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 796e1657a..70142582e 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -61,6 +61,14 @@
 HEALTHCHECK_MISMATCH_MSG = "Healthcheck pattern: %s" % HEALTHCHECK_MATCH.pattern
 
 
+def validate_port(value):
+    if not str(value).isnumeric():
+        raise serializers.ValidationError('port can only be a numeric value')
+    elif int(value) not in range(1, 65536):
+        raise serializers.ValidationError('port needs to be between 1 and 65535')
+    return value
+
+
 def validate_ptype(value):
     if not re.match(PROCTYPE_MATCH, value):
         raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
@@ -482,14 +490,6 @@ class Meta:
         fields = ['owner', 'created', 'updated', 'app', 'ptype']
         read_only_fields = ['uuid']
 
-    @staticmethod
-    def validate_port(value):
-        if not str(value).isnumeric():
-            raise serializers.ValidationError('port can only be a numeric value')
-        elif int(value) not in range(1, 65536):
-            raise serializers.ValidationError('port needs to be between 1 and 65535')
-        return value
-
     @staticmethod
     def validate_protocol(value):
         if value is None or value == "":
@@ -502,6 +502,7 @@ def validate_protocol(value):
     def validate_target_port(cls, value):
         return cls.validate_port(value)
 
+    validate_port = staticmethod(validate_port)
     validate_ptype = staticmethod(validate_ptype)
 
 
@@ -722,40 +723,28 @@ def to_representation(self, instance):
         representation['addresses'] = instance.addresses
         return representation
 
-    @staticmethod
-    def validate_port(value):
-        if not str(value).isnumeric():
-            raise serializers.ValidationError('port can only be a numeric value')
-        elif int(value) not in range(1, 65536):
-            raise serializers.ValidationError('port needs to be between 1 and 65535')
-        return value
-
     @staticmethod
     def validate_protocol(value):
         if not re.match(GATEWAY_PROTOCOL_MATCH, value):
             raise serializers.ValidationError(GATEWAY_PROTOCOL_MISMATCH_MSG)
         return value
 
+    validate_port = staticmethod(validate_port)
     validate_ptype = staticmethod(validate_ptype)
 
 
-class RouteSerializer(serializers.Serializer):
+class RouteSerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     owner = serializers.ReadOnlyField(source='owner.username')
     kind = serializers.CharField(max_length=15, required=False)
     name = serializers.CharField(max_length=63, required=True)
-    port = serializers.IntegerField()
-    ptype = serializers.CharField(max_length=63, required=True)
     rules = serializers.JSONField(required=False)
     parent_refs = serializers.JSONField(required=False)
 
-    @staticmethod
-    def validate_port(value):
-        if not str(value).isnumeric():
-            raise serializers.ValidationError('port can only be a numeric value')
-        elif int(value) not in range(1, 65536):
-            raise serializers.ValidationError('port needs to be between 1 and 65535')
-        return value
+    class Meta:
+        """Metadata options for a :class:`RouteSerializer`."""
+        model = models.gateway.Route
+        fields = '__all__'
 
     @staticmethod
     def validate_kind(value):
@@ -763,10 +752,8 @@ def validate_kind(value):
             raise serializers.ValidationError(ROUTE_PROTOCOL_MISMATCH_MSG)
         return value
 
-    validate_ptype = staticmethod(validate_ptype)
-
     def validate_rules(self, value):
-        kind = getattr(self, "initial_data", getattr(self, "data", {})).get("kind", None)
+        kind = getattr(self, "initial_data", {}).get("kind", None)
         if kind:
             schema = getattr(rules, f"{kind.replace("Route", "")}_RULES_SCHEMA", "SCHEMA")
         else:
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index 7c1bf8395..a5fc10f40 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -182,7 +182,11 @@ def gateway_changed_handle(
 def service_changed_handle(
         sender, instance: Service, created=False, update_fields=None, **kwargs):
     if kwargs['signal'] == post_delete:
-        instance.app.route_set.filter(ptype=instance.ptype).delete()
+        for route in instance.app.route_set.all():
+            if route.cleaned_rules:
+                route.save()
+            else:
+                route.delete()
 
 
 @receiver(signal=[post_save, post_delete], sender=Domain)
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index b529b260b..32e1569de 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -410,10 +410,16 @@ def create_route(self, app_id, route_name="test-route",
         response = self.client.post(
             '/v2/apps/{}/routes/'.format(app_id),
             {
-                "port": 5000,
-                "ptype": ptype,
                 "kind": kind,
                 "name": route_name,
+                "rules": [{
+                    "backendRefs": [{
+                        "kind": "Service",
+                        "name": f"{app_id}-{ptype}",
+                        "port": 5000,
+                        "weight": 100,
+                    }],
+                }],
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -433,6 +439,9 @@ def test_create_route(self):
             }
         )
         self.assertEqual(response.status_code, 404)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(response.data["count"], 1)
+        self.assertEqual(len(response.data["results"][0]["rules"]), 1)
 
     def test_hostnames(self):
         app_id = self.create_app()
@@ -493,13 +502,14 @@ def test_route_check_parent_ok(self):
         _, port, route_name = self.create_route(app_id, "myroute1", "test")
         gateway_name_1 = 'bing-gateway-1'
         self.create_gateway(app_id, gateway_name_1, 5000, "HTTP")
-        self.client.patch(
+        response = self.client.patch(
             '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
             {
                 "gateway": gateway_name_1,
                 "port": port
             }
         )
+        self.assertEqual(response.status_code, 204, response.data)
         response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
         self.assertEqual(len(response.data["results"][0]["parent_refs"]), 1)
         # create other route
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index a20330dbd..92d0a27cf 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1086,45 +1086,16 @@ def set(self, request, *args, **kwargs):
         if isinstance(rules, str):
             rules = json.loads(rules)
         rules = self.get_serializer(route, many=False).validate_rules(rules)
-        route.rules = rules
-        ok, msg = route.check_rules()
+        ok, msg = route.check_rules(rules)
         if not ok:
             return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
+        route.rules = rules
         route.save()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def create(self, request, *args, **kwargs):
-        app = self.get_app()
-        port = self.get_serializer().validate_port(request.data.get('port'))
-        app_settings = app.appsettings_set.latest()
-        ptype = self.get_serializer().validate_ptype(
-            request.data.get('ptype'))
-        kind = self.get_serializer().validate_kind(request.data.get('kind'))
-        name = request.data['name']
-        route = app.route_set.filter(name=name).first()
-        if route:
-            return Response(status=status.HTTP_400_BAD_REQUEST, data={
-                "detail": f"this route {name} already exists"})
-        route = self.model(
-            app=app,
-            owner=app.owner,
-            kind=kind,
-            name=name,
-            port=port,
-            routable=app_settings.routable,
-            ptype=ptype,
-        )
-        route.rules = route.default_rules
-        if route.rules and not route.rules[0]["backendRefs"]:
-            return Response(status=status.HTTP_400_BAD_REQUEST, data={
-                "detail": "this route does not match services. please add service first."
-            })
-        route.save()
-        return Response(status=status.HTTP_201_CREATED)
-
     def attach(self, request, *args, **kwargs):
         app = self.get_app()
-        port = self.get_serializer().validate_port(request.data.get('port'))
+        port = serializers.validate_port(request.data.get('port'))
         gateway_name = request.data['gateway']
         route = get_object_or_404(self.model, app=app, name=kwargs['name'])
         attached, msg = route.attach(gateway_name, port)
@@ -1135,7 +1106,7 @@ def attach(self, request, *args, **kwargs):
 
     def detach(self, request, *args, **kwargs):
         app = self.get_app()
-        port = self.get_serializer().validate_port(request.data.get('port'))
+        port = serializers.validate_port(request.data.get('port'))
         gateway_name = request.data['gateway']
         route = get_object_or_404(self.model, app=app, name=kwargs['name'])
         detached, msg = route.detach(gateway_name, port)

From 0be99a38453fde26d4f9550017ab555c3f42f94a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 14 Oct 2024 16:38:21 +0800
Subject: [PATCH 088/200] chore(routes): kind is required

---
 rootfs/api/models/gateway.py       | 3 ++-
 rootfs/api/serializers/__init__.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 904c4abe9..ca42d1091 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -246,7 +246,8 @@ def check_rules(self, rules):
                         has_service = True
                         break
                 if not has_service:
-                    return False, {"detail": f"service {backend_ref['name']} not exists"}
+                    msg = f"service {backend_ref['name']}:{backend_ref['port']} does not exist"
+                    return False, {"detail": msg}
         return True, ""
 
     def log(self, message, level=logging.INFO):
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 70142582e..5d53c7717 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -736,7 +736,7 @@ def validate_protocol(value):
 class RouteSerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     owner = serializers.ReadOnlyField(source='owner.username')
-    kind = serializers.CharField(max_length=15, required=False)
+    kind = serializers.CharField(max_length=15, required=True)
     name = serializers.CharField(max_length=63, required=True)
     rules = serializers.JSONField(required=False)
     parent_refs = serializers.JSONField(required=False)

From 963171df26ad2211571d1d0baff32470a79af096 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Mon, 21 Oct 2024 09:27:14 +0800
Subject: [PATCH 089/200] fix(controller): scale cover structure (#169)

---
 rootfs/api/models/app.py       | 36 +++++++++++++++++++++-------------
 rootfs/api/tests/test_build.py | 10 +++++-----
 rootfs/api/tests/test_pods.py  |  5 ++---
 3 files changed, 29 insertions(+), 22 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 251ad77b1..a7aa24430 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -16,10 +16,11 @@
 from docker import auth as docker_auth
 from django.conf import settings
 from django.db import models
+from django.db.models import F, Func, Value, JSONField
 from django.contrib.auth import get_user_model
 from rest_framework.exceptions import ValidationError, NotFound
 
-from api.utils import get_session
+from api.utils import get_session, dict_diff
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
 from api.utils import DeployLock, generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
@@ -828,29 +829,36 @@ def _scale(self, user, structure, release, app_settings):  # noqa
                 structure[target] = int(count)
             validate_app_structure(structure)
         except (TypeError, ValueError, ValidationError) as e:
-            raise DryccException('Invalid scaling format: {}'.format(e))
+            err_msg = 'Invalid scaling format: {}'.format(e)
+            self.log(err_msg)
+            raise DryccException(err_msg)
 
         # test for available process types
         for ptype in structure:
             if ptype not in (PTYPE_WEB, PTYPE_RUN) and \
                     ptype not in release.ptypes:
-                raise NotFound(
-                    'Container type {} does not exist in application'.format(ptype))
-
-        # merge current structure and the new items together
-        old_structure = self.structure
-        new_structure = old_structure.copy()
-        new_structure.update(structure)
-        if new_structure != self.structure:
+                err_msg = 'Container type {} does not exist in application'.format(ptype)
+                self.log(err_msg)
+                raise NotFound(err_msg)
+
+        new_scale = dict_diff(structure, self.structure).get("changed", {})
+        old_scale = dict_diff(self.structure, structure).get("changed", {})
+
+        if new_scale:
             try:
-                self._scale_pods(structure, release, app_settings)
+                self._scale_pods(new_scale, release, app_settings)
             except ServiceUnavailable:
                 # scaling failed, go back to old scaling numbers
-                self._scale_pods(old_structure, release, app_settings)
+                self._scale_pods(old_scale, release, app_settings)
                 raise
             # save new structure to the database
-            self.structure = new_structure
-            self.save()
+            App.objects.filter(id=self.id).update(
+                structure=Func(
+                    F("structure"),
+                    Value(new_scale, JSONField()),
+                    function="jsonb_concat",
+                )
+            )
             msg = '{} scaled pods '.format(user.username) + ' '.join(
                 "{}={}".format(k, v) for k, v in list(structure.items()))
             self.log(msg)
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 57be0485d..3bad527e7 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -259,23 +259,23 @@ def test_build_forgotten_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        # verify worker is not there
+        # verify worker is not there, only web
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 0)
+        self.assertEqual(len(response.data['results']), 1)
 
-        # verify web is not there
+        # verify web is in there
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 0)
+        self.assertEqual(len(response.data['results']), 1)
 
         # look at the app structure
         url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.json()['structure'], {'web': 0})
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
     @override_settings(DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE=False)
     def test_build_no_remove_process(self, mock_requests):
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index f9ebf32e4..2ccf94bb1 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -752,11 +752,10 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         }
         response = self.client.post(build_url, body)
         self.assertEqual(response.status_code, 201, response.data)
-
-        # make sure no pods are web
+        # web is default ptype
         application = App.objects.get(id=app_id)
         pods = application.list_pods(type='web')
-        self.assertEqual(len(pods), 0)
+        self.assertEqual(len(pods), 4)
 
     def test_list_pods_failure(self, mock_requests):
         """

From 64b4ffcb65e1d35e341f1420e8622967175c1ab0 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 21 Oct 2024 13:21:43 +0800
Subject: [PATCH 090/200] feat(release): add ptypes filter

---
 ..._route_port_remove_route_ptype_and_more.py | 26 +++++++++++++++++
 .../0017_migration_deployed_ptypes.py         | 28 +++++++++++++++++++
 rootfs/api/models/release.py                  | 23 ++++++++++++---
 rootfs/api/tests/test_release.py              | 12 +++++++-
 rootfs/api/views.py                           |  9 ++++++
 5 files changed, 93 insertions(+), 5 deletions(-)
 create mode 100644 rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py
 create mode 100644 rootfs/api/migrations/0017_migration_deployed_ptypes.py

diff --git a/rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py b/rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py
new file mode 100644
index 000000000..8f5f29df2
--- /dev/null
+++ b/rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py
@@ -0,0 +1,26 @@
+# Generated by Django 4.2.16 on 2024-10-21 04:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0015_alter_appsettings_autodeploy_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='route',
+            name='port',
+        ),
+        migrations.RemoveField(
+            model_name='route',
+            name='ptype',
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='deployed_ptypes',
+            field=models.JSONField(default=list),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0017_migration_deployed_ptypes.py b/rootfs/api/migrations/0017_migration_deployed_ptypes.py
new file mode 100644
index 000000000..25d2f7831
--- /dev/null
+++ b/rootfs/api/migrations/0017_migration_deployed_ptypes.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+
+from django.db import migrations
+from api.models.release import Release
+
+
+def migration_deployed_ptypes(apps, schema_editor):
+    for release in Release.objects.all():
+        deployed_ptypes = set()
+        for condition in release.conditions:
+            if condition["ptypes"] is None:
+                deployed_ptypes = release.ptypes
+                break
+            else:
+                deployed_ptypes = deployed_ptypes.union(condition["ptypes"])
+        release.deployed_ptypes = list(deployed_ptypes)
+        release.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0016_remove_route_port_remove_route_ptype_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_deployed_ptypes),
+    ]
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 7d4febd15..4ff02d096 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -12,7 +12,7 @@
 from scheduler import KubeHTTPException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 
-from ..utils import DeployLock
+from ..utils import CacheLock, DeployLock
 
 from .base import UuidAuditedModel
 from .appsettings import AppSettings
@@ -41,6 +41,7 @@ class Release(UuidAuditedModel):
     failed = models.BooleanField(default=False)
     exception = models.TextField(blank=True, null=True)
     conditions = models.JSONField(default=list)
+    deployed_ptypes = models.JSONField(default=list)
 
     config = models.ForeignKey('Config', on_delete=models.CASCADE)
     build = models.ForeignKey('Build', null=True, on_delete=models.CASCADE)
@@ -163,9 +164,23 @@ def get_port(self, ptype):
                 'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
 
     def deploy(self, ptypes=None, force_deploy=False):
-        lock = DeployLock(self.app.pk)
-        if not lock.acquire(ptypes, force=force_deploy):
-            raise DryccException('there is an executing pipeline, please wait or force deploy')
+        msg = 'there is an executing pipeline, please wait or force deploy'
+        # change deployed_ptypes lock
+        lock = CacheLock("release:%s" % self.pk)
+        try:
+            if lock.acquire() or force_deploy:
+                if ptypes is None:
+                    deployed_ptypes = self.ptypes
+                else:
+                    deployed_ptypes = list(set(self.deployed_ptypes).union(ptypes))
+                type(self).objects.filter(pk=self.pk).update(deployed_ptypes=deployed_ptypes)
+            else:
+                raise DryccException(msg)
+        finally:
+            lock.release()
+        # deploy lock
+        if not DeployLock(self.app.pk).acquire(ptypes, force=force_deploy):
+            raise DryccException(msg)
         run_pipeline.delay(self, ptypes, force_deploy)
 
     def previous(self):
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 2924a29b4..4c4531893 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -220,7 +220,7 @@ def test_response_data(self, mock_requests):
         for key in response.data.keys():
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'build', 'config',
                                 'summary', 'version', 'state', 'failed', 'conditions',
-                                'exception'])
+                                'exception', 'deployed_ptypes'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
@@ -260,6 +260,16 @@ def test_release_deploy(self, mock_requests):
             response = self.client.post(url, {"ptypes": "web"})
             self.assertEqual(response.status_code, 204)
 
+        url = f'/v2/apps/{app_id}/releases/?ptypes=web'
+        response = self.client.get(url)
+        self.assertEqual(response.data['count'], 1)
+        url = f'/v2/apps/{app_id}/releases/?ptypes=web,worker'
+        response = self.client.get(url)
+        self.assertEqual(response.data['count'], 1)
+        url = f'/v2/apps/{app_id}/releases/?ptypes=web,worker,noexists'
+        response = self.client.get(url)
+        self.assertEqual(response.data['count'], 0)
+
     def test_release_rollback(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 92d0a27cf..cf334d0d2 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -621,6 +621,15 @@ def get_object(self, **kwargs):
         qs = self.get_queryset(**kwargs)
         return get_object_or_404(qs, version=self.kwargs['version'])
 
+    def get_queryset(self, **kwargs):
+        ptypes = self.request.query_params.get('ptypes', '').strip()
+        queryset = super(ReleaseViewSet, self).get_queryset(**kwargs)
+        if ptypes:
+            queryset = queryset.filter(Q(
+                deployed_ptypes__contains=[
+                    ptype.lower() for ptype in re.split(r"\W+", ptypes)]))
+        return queryset
+
     def deploy(self, request, **kwargs):
         """Deploy the latest release"""
         latest_release = self.get_app().release_set.latest()

From 241765bdc43423f02835818eb406a0a695166b30 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 1 Nov 2024 14:25:38 +0800
Subject: [PATCH 091/200] chore(monitor): use memory_working_set_bytes replace
 memory_usage_bytes

---
 rootfs/api/monitor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 7b35a8bd2..244336bf8 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -73,8 +73,8 @@
   pod_name,
   container_name,
   round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  max(memory_usage_bytes) as max,
-  round(avg(memory_usage_bytes)) as avg
+  max(memory_working_set_bytes) as max,
+  round(avg(memory_working_set_bytes)) as avg
 FROM kubernetes_pod_container
 LEFT OUTER JOIN kubernetes_pod_container_tag
 ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id

From 0a9fb6fdd7d982e0c91714b5a9255e222748f6bb Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 6 Nov 2024 13:58:20 +0800
Subject: [PATCH 092/200] feat(deploy): finer grained ptype locks

---
 rootfs/api/models/app.py       | 76 ++++++++++++++++++++--------------
 rootfs/api/models/build.py     |  6 ++-
 rootfs/api/models/release.py   | 19 +++++----
 rootfs/api/tests/test_build.py | 40 +++++-------------
 rootfs/api/tests/test_lock.py  |  9 +---
 rootfs/api/tests/test_pods.py  |  6 ++-
 rootfs/api/utils.py            | 45 +++++++-------------
 7 files changed, 92 insertions(+), 109 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index a7aa24430..f295e8e62 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -249,10 +249,14 @@ def restart(self, **kwargs):  # noqa
             deployments.append(self._get_job_id(kwargs['type']))
         try:
             tasks = [
-                functools.partial(
-                    self.scheduler().deployment.restart,
-                    self.id,
-                    deployment
+                (
+                    functools.partial(
+                        self.scheduler().deployment.restart,
+                        self.id,
+                        deployment
+                    ),
+                    lambda future: self.log(
+                        f'restart {kwargs['type']} callback: {future.result()}'),
                 ) for deployment in deployments
             ]
             apply_tasks(tasks)
@@ -273,7 +277,7 @@ def scale(self, user, structure):
         app_settings = self.appsettings_set.latest()
         self._scale(user, structure, release, app_settings)
 
-    def pipeline(self, release, ptypes=None, force_deploy=False):
+    def pipeline(self, release, ptypes, force_deploy=False):
         prefix = f"[pipeline] release {release.version_name}"
         try:
             if release.build is not None:
@@ -306,7 +310,7 @@ def pipeline(self, release, ptypes=None, force_deploy=False):
                 state="crashed", action="pipeline", ptypes=ptypes, exception=str(e))
             self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
         finally:
-            DeployLock(self.pk).release(ptypes)
+            DeployLock(self.pk).release(ptypes)  # release all locks
             release.save(update_fields=["state", "failed"])  # avoid overwriting other fields
         self.log(f"{prefix} run completed...")
 
@@ -744,16 +748,20 @@ def _mount(self, user, volume, release, app_settings, structure=None):
                     'annotations', {})
                 self.set_application_config(release, scale_type)
                 # gather volume proc types to be deployed
-                tasks.append(functools.partial(
-                    self.scheduler().deployment.patch,
-                    namespace=self.id,
-                    name=self._get_job_id(scale_type),
-                    image=release.get_deploy_image(scale_type),
-                    command=release.get_deploy_command(scale_type),
-                    args=release.get_deploy_args(scale_type),
-                    spec_annotations=spec_annotations,
-                    resource_version=deployment["metadata"]["resourceVersion"],
-                    **data
+                tasks.append((
+                    functools.partial(
+                        self.scheduler().deployment.patch,
+                        namespace=self.id,
+                        name=self._get_job_id(scale_type),
+                        image=release.get_deploy_image(scale_type),
+                        command=release.get_deploy_command(scale_type),
+                        args=release.get_deploy_args(scale_type),
+                        spec_annotations=spec_annotations,
+                        resource_version=deployment["metadata"]["resourceVersion"],
+                        **data
+                    ),
+                    lambda future: self.log(
+                        f'mount {volume} for {scale_type} callback: {future.result()}'),
                 ))
         try:
             apply_tasks(tasks)
@@ -769,19 +777,24 @@ def _deploy(self, deploys, ptypes, prev_release,
         deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
         # Check if any proc type has a Deployment in progress
         self._check_deployment_in_progress(deploys, force_deploy)
-
         try:
             tasks = []
+            lock = DeployLock(self.pk)
             for scale_type, kwargs in deploys.items():
                 self.set_application_config(release, scale_type)
-                tasks.append(functools.partial(
-                    self.scheduler().deploy,
-                    namespace=self.id,
-                    name=self._get_job_id(scale_type),
-                    image=release.get_deploy_image(scale_type),
-                    command=release.get_deploy_command(scale_type),
-                    args=release.get_deploy_args(scale_type),
-                    **kwargs
+                tasks.append((
+                    functools.partial(
+                        self.scheduler().deploy,
+                        namespace=self.id,
+                        name=self._get_job_id(scale_type),
+                        image=release.get_deploy_image(scale_type),
+                        command=release.get_deploy_command(scale_type),
+                        args=release.get_deploy_args(scale_type),
+                        **kwargs
+                    ),
+                    lambda future: self.log(
+                        f'deploy and unlock callback: {[
+                            future.result(), lock.release([scale_type])]}'),
                 ))
             try:
                 apply_tasks(tasks)
@@ -797,7 +810,6 @@ def _deploy(self, deploys, ptypes, prev_release,
                     self.deploy(prev_release, ptypes, True, False)
                     # let it bubble up
                     raise DryccException('{}\n{}'.format(err, str(e))) from e
-
                 # otherwise just re-raise
                 raise
         except Exception as e:
@@ -876,7 +888,7 @@ def _scale_pods(self, scale_types, release, app_settings):
             # create the application config in k8s (secret in this case) for all deploy objects
             self.set_application_config(release, scale_type)
             # gather all proc types to be deployed
-            tasks.append(
+            tasks.append((
                 functools.partial(
                     self.scheduler().scale,
                     namespace=self.id,
@@ -885,8 +897,9 @@ def _scale_pods(self, scale_types, release, app_settings):
                     command=release.get_deploy_command(scale_type),
                     args=release.get_deploy_args(scale_type),
                     **data
-                )
-            )
+                ),
+                lambda future: self.log(f'scale {scale_type} callback: {future.result()}'),
+            ))
         try:
             apply_tasks(tasks)
         except Exception as e:
@@ -1046,10 +1059,9 @@ def _check_deployment_in_progress(self, deploys, force_deploy=False):
     @staticmethod
     def _default_structure(release):
         """Scale to default structure based on release type"""
-        structure = {PTYPE_WEB: 1}
+        structure = {}
         for ptype in release.ptypes:
-            if ptype != PTYPE_WEB:
-                structure[ptype] = 0
+            structure[ptype] = 1 if ptype == PTYPE_WEB else 0
         return structure
 
     def _scheduler_filter(self, **kwargs):
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 930989be4..35fb06c56 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -3,7 +3,7 @@
 from django.db import models
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException, Conflict
-from .base import UuidAuditedModel
+from .base import UuidAuditedModel, PTYPE_WEB
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -45,7 +45,9 @@ def type(self):
     def ptypes(self):
         if self.dryccfile:
             return list(self.dryccfile['deploy'].keys())
-        return list(self.procfile.keys())
+        if self.procfile:
+            return list(self.procfile.keys())
+        return [PTYPE_WEB]
 
     @property
     def source_based(self):
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 4ff02d096..de0cf06b6 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -164,23 +164,26 @@ def get_port(self, ptype):
                 'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
 
     def deploy(self, ptypes=None, force_deploy=False):
-        msg = 'there is an executing pipeline, please wait or force deploy'
+        ptypes = set(ptypes).intersection(self.ptypes) if ptypes else self.ptypes
+        if not ptypes:
+            self.log(f'skip deploy, ptypes are not within the optional range: {self.ptypes}')
+            return
         # change deployed_ptypes lock
+        msg = 'there is an executing pipeline, please wait or force deploy'
         lock = CacheLock("release:%s" % self.pk)
         try:
             if lock.acquire() or force_deploy:
-                if ptypes is None:
-                    deployed_ptypes = self.ptypes
-                else:
-                    deployed_ptypes = list(set(self.deployed_ptypes).union(ptypes))
-                type(self).objects.filter(pk=self.pk).update(deployed_ptypes=deployed_ptypes)
+                deployed_ptypes = list(set(self.deployed_ptypes).union(ptypes))
+                if deployed_ptypes:
+                    type(self).objects.filter(pk=self.pk).update(deployed_ptypes=deployed_ptypes)
             else:
                 raise DryccException(msg)
         finally:
             lock.release()
         # deploy lock
-        if not DeployLock(self.app.pk).acquire(ptypes, force=force_deploy):
-            raise DryccException(msg)
+        lock = DeployLock(self.app.pk)
+        if not lock.acquire(ptypes, force=force_deploy):
+            raise DryccException(f"{msg}: {lock.locked(ptypes)}")
         run_pipeline.delay(self, ptypes, force_deploy)
 
     def previous(self):
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 3bad527e7..60c5a5c7b 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -172,7 +172,7 @@ def test_build_default_containers(self, mock_requests):
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
+        self.assertEqual(response.data['results'], [])
         # start with a new app
         app_id = self.create_app()
         # post a new build with procfile
@@ -263,13 +263,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-
-        # verify web is in there
-        url = f"/v2/apps/{app_id}/pods/"
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
+        self.assertEqual([item for item in response.data['results'] if item["type"] != "web"], [])
 
         # look at the app structure
         url = f"/v2/apps/{app_id}"
@@ -319,35 +313,18 @@ def test_build_no_remove_process(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        # verify worker is still there
-        url = f"/v2/apps/{app_id}/pods/"
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertPodContains(response.data['results'], app_id, 'worker', "v3", "up")
-
-        # verify web is still there
+        # verify web
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual([item for item in response.data['results'] if item["type"] != "web"], [])
         self.assertPodContains(response.data['results'], app_id, 'web', "v3", "up")
 
         # look at the app structure
         url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.json()['structure'], {'web': 1, 'worker': 1})
-
-        # scale worker to make sure no info was lost
-        url = f"/v2/apps/{app_id}/ptypes/scale"
-        body = {'worker': 2}  # bump from 1 to 2
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # verify worker info
-        url = f"/v2/apps/{app_id}/pods/"
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertPodContains(response.data['results'], app_id, 'worker', "v3", "up")
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
     @override_settings(DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING=True)
     def test_build_forgotten_procfile_reject(self, mock_requests):
@@ -389,7 +366,12 @@ def test_build_forgotten_procfile_reject(self, mock_requests):
         url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 409, response.data)
+        self.assertEqual(response.status_code, 201, response.data)
+        # verify web
+        url = f"/v2/apps/{app_id}/pods/"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertPodContains(response.data['results'], app_id, 'web', "v3", "up")
 
     def test_build_str(self, mock_requests):
         """Test the text representation of a build."""
diff --git a/rootfs/api/tests/test_lock.py b/rootfs/api/tests/test_lock.py
index 85dc76858..146c8f037 100644
--- a/rootfs/api/tests/test_lock.py
+++ b/rootfs/api/tests/test_lock.py
@@ -25,24 +25,19 @@ def test_deploy_lock(self):
         app_id = f"test_key_1_{int(time.time())}"
         lock1 = DeployLock(app_id)
         lock2 = DeployLock(app_id)
-        self.assertEqual(lock1.acquire(None), True)
-        self.assertEqual(lock2.acquire(["web", "task"]), False)
-        lock1.release(None)
         self.assertEqual(lock2.acquire(["web", "task"]), True)
-        self.assertEqual(lock1.acquire(None), False)
         self.assertEqual(lock1.acquire(["web"]), False)
         self.assertEqual(lock1.acquire(["bing"]), True)
         lock2.release(["web", "task"])
         self.assertEqual(lock1.acquire(["web"]), True)
         self.assertEqual(lock1.acquire(["task"]), True)
         self.assertEqual(lock2.acquire(["web", "task"]), False)
-        self.assertEqual(lock2.acquire(None), False)
         lock2.release(["web", "task", "bing"])
-        self.assertEqual(lock2.acquire(None), True)
-        lock2.release(None)
         self.assertEqual(lock1.acquire(["web", "task"]), True)
         self.assertEqual(lock2.acquire(["web", "task"]), False)
         self.assertEqual(lock2.acquire(["web", "bing"], True), True)
         self.assertEqual(lock1.acquire(["web", "bing", "task"]), False)
         self.assertEqual(lock1.acquire(["web", "bing", "task"], True), True)
         lock2.release(["web", "bing", "task"])
+        self.assertEqual(lock1.acquire(["web"], True), True)
+        lock1.release(["web", "bing", "task"])
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index 2ccf94bb1..b88592e30 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -752,10 +752,12 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         }
         response = self.client.post(build_url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        # web is default ptype
+        # check web pods
         application = App.objects.get(id=app_id)
         pods = application.list_pods(type='web')
-        self.assertEqual(len(pods), 4)
+        self.assertEqual(len(pods), 0)
+        pods = application.list_pods(type='worker')
+        self.assertEqual(len(pods), 0)
 
     def test_list_pods_failure(self, mock_requests):
         """
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index 96f3d57d2..0c0621262 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -160,7 +160,8 @@ def apply_tasks(tasks):
         return
 
     executor = concurrent.futures.ThreadPoolExecutor(5)
-    for future in [executor.submit(task) for task in tasks]:
+    for future, callback in [(executor.submit(task[0]), task[1]) for task in tasks]:
+        future.add_done_callback(callback)
         error = future.exception()
         if error is not None:
             raise error
@@ -231,32 +232,20 @@ def __init__(self, app_id):
 
     def locked(self, ptypes):
         value = cache.get(self.ptypes_key, [])
-        # None will locks all
-        if ptypes is None:
-            if value is None or len(value) > 0:
-                return True
-            elif len(value) == 0:
-                return False
-        else:
-            if value is None:
-                return True
-            for ptype in ptypes:
-                if ptype in value:
-                    return True
-        return False
+        locked_ptypes = []
+        for ptype in ptypes:
+            if ptype in value:
+                locked_ptypes.append(ptype)
+        return locked_ptypes
 
     def acquire(self, ptypes, force=False):
         try:
             if super(DeployLock, self).acquire():
                 value = cache.get(self.ptypes_key, [])
-                # None will locks all
-                if not force and self.locked(ptypes):
+                if not force and len(self.locked(ptypes)) > 0:
                     return False
-                if ptypes is None:
-                    value = None
-                else:
-                    value.extend(ptypes)
-                    value = list(set(value))
+                value.extend(ptypes)
+                value = list(set(value))
                 cache.set(self.ptypes_key, value, timeout=3600)
                 return True
         finally:
@@ -266,15 +255,13 @@ def acquire(self, ptypes, force=False):
     def release(self, ptypes):
         try:
             if super(DeployLock, self).acquire():
-                if ptypes is None:
-                    cache.delete(self.ptypes_key)
-                else:
-                    value = cache.get(self.ptypes_key, [])
-                    if value is None:
-                        return
-                    for ptype in ptypes:
+                value = cache.get(self.ptypes_key, [])
+                if value is None:
+                    return
+                for ptype in ptypes:
+                    if ptype in value:
                         value.remove(ptype)
-                    cache.set(self.ptypes_key, value)
+                cache.set(self.ptypes_key, value)
         finally:
             super(DeployLock, self).release()
 

From ba190f4f8e1654b87652534efce5adc5dcbe05d3 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 8 Nov 2024 18:44:55 +0800
Subject: [PATCH 093/200] feat(config): add config for dryccfile

---
 rootfs/api/admissions.py                      |  58 ++-
 .../0017_migration_deployed_ptypes.py         |   2 +-
 .../api/migrations/0018_config_values_refs.py |  18 +
 ...igration_config_values_and_typed_values.py |  38 ++
 ...config_typed_values_alter_config_values.py |  22 +
 rootfs/api/models/app.py                      | 135 +++---
 rootfs/api/models/appsettings.py              |  25 +-
 rootfs/api/models/build.py                    |  92 ++--
 rootfs/api/models/config.py                   | 197 ++++++---
 rootfs/api/models/release.py                  |  59 ++-
 rootfs/api/serializers/__init__.py            |  52 ++-
 rootfs/api/serializers/schemas/config.py      |  30 ++
 rootfs/api/settings/production.py             |  23 -
 rootfs/api/tests/test_app.py                  |  22 +-
 rootfs/api/tests/test_build.py                | 109 +++--
 rootfs/api/tests/test_config.py               | 418 ++++++++++++++----
 rootfs/api/tests/test_pods.py                 |   3 +-
 rootfs/api/tests/test_registry.py             |  56 +--
 rootfs/api/tests/test_release.py              | 203 ++++++++-
 rootfs/api/tests/test_tags.py                 |   2 -
 rootfs/api/urls.py                            |   4 +
 rootfs/api/utils.py                           |   2 +-
 rootfs/api/views.py                           |  38 +-
 rootfs/scheduler/resources/secret.py          |   4 +-
 24 files changed, 1128 insertions(+), 484 deletions(-)
 create mode 100644 rootfs/api/migrations/0018_config_values_refs.py
 create mode 100644 rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
 create mode 100644 rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py
 create mode 100644 rootfs/api/serializers/schemas/config.py

diff --git a/rootfs/api/admissions.py b/rootfs/api/admissions.py
index 48b050faf..859f9b847 100644
--- a/rootfs/api/admissions.py
+++ b/rootfs/api/admissions.py
@@ -30,22 +30,27 @@ def handle(self, request: Request) -> bool:
         app = models.app.App.objects.filter(id=app_id).first()
         ptype = request["object"]["metadata"].get("labels", {}).get("type", "")
         if app and ptype:
-            status = request["object"]["status"]
-            replicas = request["object"]["spec"].get("replicas", 0)
-            if "active" in status:
-                replicas += 1
-            elif "succeeded" in status or "failed" in status:
-                replicas -= 1
-            replicas = 0 if replicas < 0 else replicas
-            if app.structure.get(ptype, 0) != replicas:
-                models.app.App.objects.filter(id=app.id).update(
-                    structure=Func(
-                        F("structure"),
-                        Value([ptype]),
-                        Value(replicas, JSONField()),
-                        function="jsonb_set",
+            lock = app.lock()
+            try:
+                lock.acquire()
+                status = request["object"]["status"]
+                replicas = request["object"]["spec"].get("replicas", 0)
+                if "active" in status:
+                    replicas += 1
+                elif "succeeded" in status or "failed" in status:
+                    replicas -= 1
+                replicas = 0 if replicas < 0 else replicas
+                if app.structure.get(ptype, 0) != replicas:
+                    models.app.App.objects.filter(id=app.id).update(
+                        structure=Func(
+                            F("structure"),
+                            Value([ptype]),
+                            Value(replicas, JSONField()),
+                            function="jsonb_set",
+                        )
                     )
-                )
+            finally:
+                lock.release()
         return True
 
 
@@ -70,16 +75,21 @@ def handle(self, request: Request) -> bool:
             if key == "type":
                 ptype = value
         if app and ptype:
-            replicas = request["object"]["spec"].get("replicas", 0)
-            if app.structure.get(ptype, 0) != replicas:
-                models.app.App.objects.filter(id=app.id).update(
-                    structure=Func(
-                        F("structure"),
-                        Value([ptype]),
-                        Value(replicas, JSONField()),
-                        function="jsonb_set",
+            lock = app.lock()
+            try:
+                lock.acquire()
+                replicas = request["object"]["spec"].get("replicas", 0)
+                if app.structure.get(ptype, 0) != replicas:
+                    models.app.App.objects.filter(id=app.id).update(
+                        structure=Func(
+                            F("structure"),
+                            Value([ptype]),
+                            Value(replicas, JSONField()),
+                            function="jsonb_set",
+                        )
                     )
-                )
+            finally:
+                lock.release()
         return True
 
 
diff --git a/rootfs/api/migrations/0017_migration_deployed_ptypes.py b/rootfs/api/migrations/0017_migration_deployed_ptypes.py
index 25d2f7831..197f82d7c 100644
--- a/rootfs/api/migrations/0017_migration_deployed_ptypes.py
+++ b/rootfs/api/migrations/0017_migration_deployed_ptypes.py
@@ -5,7 +5,7 @@
 
 
 def migration_deployed_ptypes(apps, schema_editor):
-    for release in Release.objects.all():
+    for release in Release.objects.all().using("default"):
         deployed_ptypes = set()
         for condition in release.conditions:
             if condition["ptypes"] is None:
diff --git a/rootfs/api/migrations/0018_config_values_refs.py b/rootfs/api/migrations/0018_config_values_refs.py
new file mode 100644
index 000000000..abb11eead
--- /dev/null
+++ b/rootfs/api/migrations/0018_config_values_refs.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-11-12 02:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0017_migration_deployed_ptypes'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='config',
+            name='values_refs',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py b/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
new file mode 100644
index 000000000..e96cbe812
--- /dev/null
+++ b/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
@@ -0,0 +1,38 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+import json
+from django.db import connection, migrations
+from api.models.config import Config
+
+
+def migration_values(apps, schema_editor):
+    with connection.cursor() as cursor:
+        cursor.execute("SELECT uuid,values,typed_values FROM api_config")
+        for uuid, values, typed_values in cursor:
+            new_values = []
+            config = Config.objects.using('default').get(pk=uuid)
+            for name, value in json.loads(values).items():
+                new_values.append({
+                    "name": name,
+                    "group": "global",
+                    "value": value
+                })
+            for ptype, values in json.loads(typed_values).items():
+                for name, value in values.items():
+                    new_values.append({
+                        "name": name,
+                        "ptype": ptype,
+                        "value": value
+                    })
+            config.values = new_values
+            config.save(ignore_update_fields=config.allof_fields)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0018_config_values_refs'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_values),
+    ]
diff --git a/rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py b/rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py
new file mode 100644
index 000000000..78e0eadfd
--- /dev/null
+++ b/rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.16 on 2024-11-06 06:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0019_migration_config_values_and_typed_values'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='config',
+            name='typed_values',
+        ),
+        migrations.AlterField(
+            model_name='config',
+            name='values',
+            field=models.JSONField(blank=True, default=list),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index f295e8e62..426637cbb 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -18,11 +18,11 @@
 from django.db import models
 from django.db.models import F, Func, Value, JSONField
 from django.contrib.auth import get_user_model
-from rest_framework.exceptions import ValidationError, NotFound
+from rest_framework.exceptions import ValidationError
 
 from api.utils import get_session, dict_diff
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
-from api.utils import DeployLock, generate_app_name, apply_tasks
+from api.utils import CacheLock, DeployLock, generate_app_name, apply_tasks
 from scheduler import KubeHTTPException, KubeException
 from .gateway import Gateway, Route
 from .limit import LimitPlan
@@ -148,6 +148,9 @@ def save(self, *args, **kwargs):
 
         return application
 
+    def lock(self):
+        return CacheLock(f"app:lock:{self.id}")
+
     @property
     def ptypes(self):
         return list(self.structure.keys())
@@ -246,7 +249,7 @@ def restart(self, **kwargs):  # noqa
         """
         deployments = []
         if self.structure[kwargs['type']] > 0:
-            deployments.append(self._get_job_id(kwargs['type']))
+            deployments.append(self._get_deployment_name(kwargs['type']))
         try:
             tasks = [
                 (
@@ -326,16 +329,7 @@ def deploy(self, release, ptypes=None, force_deploy=False, rollback_on_failure=T
         self.create()
         # Previous release
         prev_release = release.previous()
-
-        default_structure = self._default_structure(release)
-        if (self.structure != default_structure) or (
-            prev_release and prev_release.build and prev_release.build.type != release.build.type
-        ):
-            # structure {} or build type change, merge old structure if exists
-            for ptype, scale in default_structure.items():
-                default_structure[ptype] = self.structure.get(ptype, scale)
-            self.structure = default_structure
-            self.save()
+        self._merge_structure(release, prev_release)
         # deploy application to k8s. Also handles initial scaling
         app_settings = self.appsettings_set.latest()
         volumes = self.volume_set.all()
@@ -349,8 +343,9 @@ def deploy(self, release, ptypes=None, force_deploy=False, rollback_on_failure=T
         self._deploy(
             deploys, ptypes, prev_release, release, force_deploy, rollback_on_failure)
         # cleanup old release objects from kubernetes
-        self.cleanup_old(ptypes)
-        release.cleanup_old(ptypes)
+        if app_settings.autodeploy:
+            self.clean(release=release)
+        release.clean(ptypes)
 
     def mount(self, user, volume, structure=None):
         if self.release_set.filter(failed=False).latest().build is None:
@@ -359,20 +354,32 @@ def mount(self, user, volume, structure=None):
         app_settings = self.appsettings_set.latest()
         self._mount(user, volume, release, app_settings, structure=structure)
 
-    def cleanup_old(self, ptypes=None):
-        names = []
-        for scale_type in self.structure.keys():
-            names.append(self._get_job_id(scale_type))
+    def clean(self, release=None, ptypes=None):
+        release = release if release else self.release_set.latest()
+        # scale ptype down to 0, the next deploy will delete
+        pre_release = release.previous(None)
+        if pre_release is not None:
+            removed = {}
+            for ptype in pre_release.ptypes:
+                if ptype not in release.ptypes and self.structure.get(ptype, 0) > 0:
+                    if ptypes is None or ptype in ptypes:
+                        removed[ptype] = 0
+            self.scale(self.owner, removed)
+        self._merge_structure(release, pre_release)
+        # clean k8s resources, ptype not in structure
         labels = {'heritage': 'drycc'}
-        if ptypes is not None:
+        if ptypes:
             labels["type__in"] = ptypes
-        deployments = self.scheduler().deployments.get(self.id, labels=labels).json()["items"]
-        if deployments is not None:
-            for deployment in deployments:
-                name = deployment['metadata']['name']
-                if name not in names:
-                    self.scheduler().deployments.delete(self.id, name, True)
-        self.log(f"cleanup old kubernetes deployments for {self.id}")
+        resource_apis = [self.scheduler().deployments, self.scheduler().secret]
+        for api in resource_apis:
+            resources = api.get(self.id, labels=labels).json()["items"]
+            if resources is not None:
+                for resource in resources:
+                    name = resource['metadata']['name']
+                    ptype = resource['metadata'].get("labels", {}).get("type")
+                    if (ptype and ptype not in self.structure):
+                        api.delete(self.id, name, True)
+        self.log(f"cleanup old kubernetes resources for {self.id}")
 
     def run(self, user, image=None, command=None, args=None, volumes=None,
             timeout=3600, expires=3600, **kwargs):
@@ -400,7 +407,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         data['restart_policy'] = 'Never'
         data['active_deadline_seconds'] = timeout
         data['ttl_seconds_after_finished'] = expires
-        name = self._get_job_id(PTYPE_RUN) + '-' + pod_name()
+        name = self._get_deployment_name(PTYPE_RUN) + '-' + pod_name()
         self.log("{} on {} runs '{}'".format(user.username, name, command))
         kwargs.update(data)
         try:
@@ -521,6 +528,7 @@ def describe_deployment(self, deployment_name):
 
     def list_deployments(self, *args, **kwargs):
         """Used to list basic information about deployments running for a given application"""
+        ptypes = self.release_set.latest().ptypes
         try:
             labels = self._scheduler_filter(**kwargs)
             # in case a singular deployment is requested
@@ -545,6 +553,7 @@ def list_deployments(self, *args, **kwargs):
                         p["status"].get("readyReplicas", 0),
                         p["status"].get("replicas", 0),
                     ),
+                    'garbage': False if labels['type'] in ptypes else True,
                     'up_to_date': p["status"].get("updatedReplicas", 0),
                     'available_replicas': p["status"].get("availableReplicas", 0),
                     'started': started
@@ -654,7 +663,7 @@ def state_to_k8s(self):
         ptypes = set()
         for ptype, scale in self.structure.items():
             response = self.scheduler().deployment.get(
-                self.id, self._get_job_id(ptype),
+                self.id, self._get_deployment_name(ptype),
                 ignore_exception=True)
             if response.status_code == 404 and scale > 0:
                 ptypes.add(ptype)
@@ -717,7 +726,7 @@ def to_measurements(self, timestamp: float):
     def __str__(self):
         return self.id
 
-    def _get_job_id(self, ptype):
+    def _get_deployment_name(self, ptype):
         return f"{self.id}-{ptype}"
 
     def _clean_app_logs(self):
@@ -743,7 +752,7 @@ def _mount(self, user, volume, release, app_settings, structure=None):
                 data = self._gather_app_settings(
                     release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
                 deployment = self.scheduler().deployment.get(
-                    self.id, self._get_job_id(scale_type)).json()
+                    self.id, self._get_deployment_name(scale_type)).json()
                 spec_annotations = deployment['spec']['template']['metadata'].get(
                     'annotations', {})
                 self.set_application_config(release, scale_type)
@@ -752,7 +761,7 @@ def _mount(self, user, volume, release, app_settings, structure=None):
                     functools.partial(
                         self.scheduler().deployment.patch,
                         namespace=self.id,
-                        name=self._get_job_id(scale_type),
+                        name=self._get_deployment_name(scale_type),
                         image=release.get_deploy_image(scale_type),
                         command=release.get_deploy_command(scale_type),
                         args=release.get_deploy_args(scale_type),
@@ -786,7 +795,7 @@ def _deploy(self, deploys, ptypes, prev_release,
                     functools.partial(
                         self.scheduler().deploy,
                         namespace=self.id,
-                        name=self._get_job_id(scale_type),
+                        name=self._get_deployment_name(scale_type),
                         image=release.get_deploy_image(scale_type),
                         command=release.get_deploy_command(scale_type),
                         args=release.get_deploy_args(scale_type),
@@ -845,14 +854,6 @@ def _scale(self, user, structure, release, app_settings):  # noqa
             self.log(err_msg)
             raise DryccException(err_msg)
 
-        # test for available process types
-        for ptype in structure:
-            if ptype not in (PTYPE_WEB, PTYPE_RUN) and \
-                    ptype not in release.ptypes:
-                err_msg = 'Container type {} does not exist in application'.format(ptype)
-                self.log(err_msg)
-                raise NotFound(err_msg)
-
         new_scale = dict_diff(structure, self.structure).get("changed", {})
         old_scale = dict_diff(self.structure, structure).get("changed", {})
 
@@ -892,7 +893,7 @@ def _scale_pods(self, scale_types, release, app_settings):
                 functools.partial(
                     self.scheduler().scale,
                     namespace=self.id,
-                    name=self._get_job_id(scale_type),
+                    name=self._get_deployment_name(scale_type),
                     image=release.get_deploy_image(scale_type),
                     command=release.get_deploy_command(scale_type),
                     args=release.get_deploy_args(scale_type),
@@ -1046,7 +1047,7 @@ def _check_deployment_in_progress(self, deploys, force_deploy=False):
         if force_deploy:
             return
         for scale_type, kwargs in deploys.items():
-            name = self._get_job_id(scale_type)
+            name = self._get_deployment_name(scale_type)
             # Is there an existing deployment in progress?
             in_progress, deploy_okay = self.scheduler().deployment.in_progress(
                 self.id, name, kwargs.get("deploy_timeout"), kwargs.get("deploy_batches"),
@@ -1056,13 +1057,29 @@ def _check_deployment_in_progress(self, deploys, force_deploy=False):
             if in_progress and not deploy_okay:
                 raise AlreadyExists('Deployment for {} is already in progress'.format(name))
 
-    @staticmethod
-    def _default_structure(release):
+    def _merge_structure(self, release, prev_release):
         """Scale to default structure based on release type"""
-        structure = {}
-        for ptype in release.ptypes:
-            structure[ptype] = 1 if ptype == PTYPE_WEB else 0
-        return structure
+        lock = self.lock()
+        try:
+            lock.acquire()
+            self.refresh_from_db()
+            default_structure = {}
+            for ptype in release.ptypes:
+                default_structure[ptype] = 1 if ptype == PTYPE_WEB else 0
+            if (self.structure != default_structure) or (
+                prev_release and prev_release.build and
+                prev_release.build.type != release.build.type
+            ):
+                for ptype, scale in self.structure.items():
+                    # clean old ptype
+                    if ptype not in release.ptypes and scale == 0:
+                        continue
+                    default_structure[ptype] = scale
+                self.structure = default_structure
+                self.save()
+        finally:
+            lock.release()
+        return self.structure
 
     def _scheduler_filter(self, **kwargs):
         labels = {'app': self.id, 'heritage': 'drycc'}
@@ -1095,8 +1112,7 @@ def _build_env_vars(self, release, ptype):
 
         default_env['SOURCE_VERSION'] = release.build.sha
         # merge envs on top of default to make envs win
-        default_env.update(release.config.values)
-        default_env.update(release.config.typed_values.get(ptype, {}))
+        default_env.update(release.config.envs(ptype))
         # fetch application port and inject into ENV vars as needed
         port = release.get_port(ptype)
         if port:
@@ -1168,28 +1184,29 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
         limit_plan = LimitPlan.objects.get(id=config.limits.get(ptype))
 
         # see if the app config has deploy batch preference, otherwise use global
-        batches = int(config.values.get('DRYCC_DEPLOY_BATCHES', settings.DRYCC_DEPLOY_BATCHES))
+        batches = int(envs.get('DRYCC_DEPLOY_BATCHES', settings.DRYCC_DEPLOY_BATCHES))
 
         # see if the app config has deploy timeout preference, otherwise use global
-        deploy_timeout = int(
-            config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT))
+        deploy_timeout = int(envs.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT))
 
         # configures how many ReplicaSets to keep beside the latest version
-        deployment_history = config.values.get(
+        deployment_history = envs.get(
             'KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT',
             settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT)
 
         # get application level pod termination grace period
-        pod_termination_grace_period_seconds = int(config.values.get(
+        pod_termination_grace_period_seconds = int(envs.get(
             'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS',
             settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS))
 
         # set the image pull policy that is associated with the application container
-        image_pull_policy = config.values.get('IMAGE_PULL_POLICY', settings.IMAGE_PULL_POLICY)
+        image_pull_policy = envs.get('IMAGE_PULL_POLICY', settings.IMAGE_PULL_POLICY)
 
+        # set registry
+        registry = config.registry.get(ptype, {})
         # create image pull secret if needed
         image_pull_secret_name = self.image_pull_secret(
-            self.id, config.registry, release.get_deploy_image(ptype))
+            self.id, registry, release.get_deploy_image(ptype))
 
         # only web is routable
         # https://www.drycc.cc/applications/managing-app-processes/#default-process-types
@@ -1201,7 +1218,7 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
         return {
             'tags': config.tags.get(ptype, {}),
             'envs': envs,
-            'registry': config.registry,
+            'registry': registry,
             'replicas': replicas,
             'version': release.version_name,
             'app_type': ptype,
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index 46d00a035..9a9bf194d 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -1,4 +1,6 @@
 import logging
+
+from functools import partial
 from django.db import models
 from django.db import transaction
 
@@ -147,7 +149,8 @@ def _update_label(self, previous_settings):
                     self.summary += ' and '
                 self.summary += ["{} {}".format(self.owner, changes)]
 
-    def _update_fields(self, ignore_update_fields=None):
+    @transaction.atomic
+    def save(self, ignore_update_fields=None, *args, **kwargs):
         previous_settings = None
         try:
             previous_settings = self.app.appsettings_set.latest()
@@ -155,13 +158,13 @@ def _update_fields(self, ignore_update_fields=None):
             pass
         update_fields = ["routable", "autodeploy", "autorollback", "autoscale", "label"]
         try:
-            for update_field in update_fields:
-                if ignore_update_fields is None or update_field not in ignore_update_fields:
-                    method = getattr(self, "_update_%s" % update_field, None)
-                    if method:
-                        method(previous_settings)
-                    else:
-                        self._update_field(update_field, previous_settings)
+            for field in update_fields:
+                if ignore_update_fields is None or field not in ignore_update_fields:
+                    getattr(
+                        self,
+                        "_update_%s" % field,
+                        partial(self._update_field, field)
+                    )(previous_settings)
         except (UnprocessableEntity, NotFound):
             raise
         except Exception as e:
@@ -171,10 +174,6 @@ def _update_fields(self, ignore_update_fields=None):
         if not self.summary and previous_settings:
             self.delete()
             raise AlreadyExists("{} changed nothing".format(self.owner))
+        super(AppSettings, self).save(**kwargs)
         summary = ' '.join(self.summary)
         self.log('summary of app setting changes: {}'.format(summary), logging.DEBUG)
-
-    @transaction.atomic
-    def save(self, ignore_update_field=None, *args, **kwargs):
-        self._update_fields(ignore_update_field)
-        super(AppSettings, self).save(**kwargs)
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 35fb06c56..f00b2b58f 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -1,9 +1,9 @@
 import logging
-from django.conf import settings
 from django.db import models
 from django.contrib.auth import get_user_model
-from api.exceptions import DryccException, Conflict
+from api.exceptions import DryccException
 from .base import UuidAuditedModel, PTYPE_WEB
+from .config import Config
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -73,18 +73,18 @@ def get_image(self, ptype, default_image=None):
 
     def create_release(self, user, *args, **kwargs):
         latest_release = self.app.release_set.filter(failed=False).latest()
-        latest_version = self.app.release_set.latest().version
         try:
             new_release = latest_release.new(
                 user,
                 build=self,
-                config=latest_release.config,
+                config=self._get_or_create_config(),
             )
             if self.app.appsettings_set.latest().autodeploy:
                 new_release.deploy(force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
+            latest_version = self.app.release_set.latest().version
             if ('new_release' not in locals() and
                     self.app.release_set.latest().version == latest_version+1):
                 new_release = self.app.release_set.latest()
@@ -102,59 +102,35 @@ def create_release(self, user, *args, **kwargs):
                 self.delete()
             raise DryccException(str(e)) from e
 
-    def save(self, **kwargs):
-        previous_release = self.app.release_set.filter(failed=False).latest()
-
-        if (
-            settings.DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING is True and
-            # previous release had a Procfile and the current one does not
-            (
-                previous_release.build is not None and
-                len(previous_release.ptypes) > 0 and
-                len(self.ptypes) == 0
-            )
-        ):
-            # Reject deployment
-            raise Conflict(
-                'Last deployment had process types but is missing in this deploy. '
-                'For a successful deployment provide process types.'
-            )
-
-        # See if processes are permitted to be removed
-        remove_procs = (
-            # If set to True then contents of Procfile does not affect the outcome
-            settings.DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE is True or
-            # previous release had a Procfile and the current one does as well
-            (
-                previous_release.build is not None and
-                len(previous_release.ptypes) > 0 and
-                len(self.ptypes) > 0
-            )
-        )
-
-        # spin down any proc type removed between the last procfile and the newest one
-        if remove_procs and previous_release.build is not None and \
-                self.app.appsettings_set.latest().autodeploy:
-            removed = {}
-            for proc in previous_release.ptypes:
-                if proc not in self.ptypes and self.app.structure.get(proc, 0) > 0:
-                    # Scale proc type down to 0
-                    removed[proc] = 0
-
-            self.app.scale(self.owner, removed)
-
-        # make sure the latest build has procfile if the intent is to
-        # allow empty Procfile without removals
-        if (
-            settings.DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE is False and
-            previous_release.build is not None and
-            len(previous_release.ptypes) > 0 and
-            len(self.ptypes) == 0
-        ):
-            self.procfile = previous_release.build.procfile
-            self.dryccfile = previous_release.build.dryccfile
-
-        return super(Build, self).save(**kwargs)
-
     def __str__(self):
         return "{0}-{1}".format(self.app.id, str(self.uuid)[:7])
+
+    def _get_or_create_config(self):
+        """
+        dryccfile to config
+        """
+        latest_release = self.app.release_set.filter(failed=False).latest()
+        config_values, config_values_ref, config_healthcheck = [], {}, {}
+        for group, values in self.dryccfile.get('config', {}).items():
+            for value in values:
+                value['group'] = group
+                config_values.append(value)
+        for ptype, values in self.dryccfile.get('deploy', {}).items():
+            for value in values.get('config', {}).get('env', []):
+                value['ptype'] = ptype
+                config_values.append(value)
+            for config_ref in values.get('config', {}).get('ref', []):
+                if ptype not in config_values_ref:
+                    config_values_ref[ptype] = [config_ref]
+                else:
+                    config_values_ref[ptype].append(config_ref)
+            if 'healthcheck' in values:
+                config_healthcheck[ptype] = values.get('healthcheck')
+        if not config_values:
+            return latest_release.config
+        config = Config(
+            owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref,
+            healthcheck=config_healthcheck,
+        )
+        config.save(ignore_update_fields=["values", "values_refs", "healthcheck"])
+        return config
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index e28633ef2..afe0b8552 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -1,4 +1,5 @@
 import logging
+from functools import partial
 from django.db import models
 from django.contrib.auth import get_user_model
 from api.utils import dict_diff
@@ -16,13 +17,13 @@ class Config(UuidAuditedModel):
     during runtime execution of the Application.
     """
     ptype_fields = ("lifecycle_post_start", "lifecycle_pre_stop", "tags", "limits",
-                    "typed_values", "healthcheck", "termination_grace_period")
-    allof_fields = ("values", "registry") + ptype_fields
+                    "values_refs", "healthcheck", "termination_grace_period", "registry")
+    allof_fields = ("values", ) + ptype_fields
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    values = models.JSONField(default=dict, blank=True)
-    typed_values = models.JSONField(default=dict, blank=True)
+    values = models.JSONField(default=list, blank=True)
+    values_refs = models.JSONField(default=dict, blank=True)
     lifecycle_post_start = models.JSONField(default=dict, blank=True)
     lifecycle_pre_stop = models.JSONField(default=dict, blank=True)
     tags = models.JSONField(default=dict, blank=True)
@@ -56,14 +57,71 @@ def previous(self):
             prev_config = None
         return prev_config
 
-    def diff(self, config=None):
-        old_config = config if config else self.previous()
+    def envs(self, ptype):
+        envs = {}
+        # group env
+        groups = ['global']
+        groups.extend(self.values_refs.get(ptype, []))
+        for group in groups:
+            for value in self.values:
+                if group == value.get('group'):
+                    envs[value['name']] = value['value']
+        # ptype env, repetition will replace the previous env
+        for value in self.values:
+            if value.get('ptype') == ptype:
+                envs[value['name']] = value['value']
+        return envs
+
+    def diff(self, old_config):
+        def flat_values(values):
+            ptype_envs, group_envs = {}, {}
+            for value in values:
+                ptype, group = value.get('ptype'), value.get('group')
+                if ptype:
+                    ptype_envs[ptype] = ptype_envs.get(ptype, {})
+                    ptype_envs[ptype][value['name']] = value['value']
+                if group:
+                    group_envs[group] = group_envs.get(group, {})
+                    group_envs[group][value['name']] = value['value']
+            return ptype_envs, group_envs
+
+        old_config = old_config if old_config else self.previous()
+        # ptype field diff
         result = {}
-        for field in self.allof_fields:
-            result[field] = dict_diff(getattr(self, field), getattr(old_config, field))
+        for field in self.ptype_fields:
+            new_value = getattr(self, field)
+            old_value = getattr(old_config, field)
+            result[field] = dict_diff(new_value, old_value)
+        values_diff = {}
+        (new_ptype_envs, new_group_envs), (old_ptype_envs, old_group_envs) = (
+            flat_values(self.values), flat_values(old_config.values))
+        # diff ptype env
+        for ptype in set(new_ptype_envs.keys()).union(old_ptype_envs.keys()):
+            values_diff.update(
+                dict_diff(new_ptype_envs.get(ptype, {}), old_ptype_envs.get(ptype, {})))
+        # diff group env
+        for group in set(new_group_envs.keys()).union(old_group_envs.keys()):
+            values_diff.update(
+                dict_diff(new_group_envs.get(group, {}), old_group_envs.get(group, {})))
+        result["values"] = values_diff
         return result
 
-    def save(self, **kwargs):
+    def diff_ptypes(self, old_config, include_ptypes):
+        old_config = old_config if old_config else self.previous()
+        ptypes = set()
+        for ptype in include_ptypes:
+            for field in self.ptype_fields:
+                new_value = getattr(self, field).get(ptype, None)
+                old_value = getattr(old_config, field).get(ptype, None)
+                if (new_value or old_value) and new_value != old_value:
+                    ptypes.add(ptype)
+            new_env = self.envs(ptype)
+            old_env = old_config.envs(ptype)
+            if new_env != old_env:
+                ptypes.add(ptype)
+        return ptypes
+
+    def save(self, ignore_update_fields=None, *args, **kwargs):
         """merge the old config with the new"""
         try:
             # Get config from the latest available release
@@ -73,22 +131,69 @@ def save(self, **kwargs):
                 # If that doesn't exist then fallback on app config
                 # usually means a totally new app
                 previous_config = self.app.config_set.latest()
-
-            for attr in ['registry', 'values', 'lifecycle_post_start',
-                         'lifecycle_pre_stop', 'termination_grace_period']:
-                data = getattr(previous_config, attr, {}).copy()
-                new_data = getattr(self, attr, {}).copy()
-                self._merge_data(attr, data, new_data)
-                setattr(self, attr, data)
-            self._set_typed_values(previous_config)
-            self._set_limits(previous_config)
-            self._set_healthcheck(previous_config)
-            self._set_registry()
-            self._set_tags(previous_config)
+            for field in self.allof_fields:
+                if ignore_update_fields is None or field not in ignore_update_fields:
+                    getattr(
+                        self,
+                        "_update_%s" % field,
+                        partial(self._update_field, field)
+                    )(previous_config)
         except Config.DoesNotExist:
-            self._set_tags(previous_config={'tags': {}})
+            self._update_tags(previous_config={'tags': {}})
+        return super(Config, self).save(*args, **kwargs)
 
-        return super(Config, self).save(**kwargs)
+    def _update_field(self, field, previous_config):
+        data = getattr(previous_config, field, {}).copy()
+        new_data = getattr(self, field, {}).copy()
+        # remove config keys if a null value is provided
+        for key, value in new_data.items():
+            if value is None:
+                # error if unsetting non-existing key
+                if key not in data:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(key, field))
+                data.pop(key)
+            else:
+                data[key] = self._merge_data(field, data.get(key, {}), value)
+        setattr(self, field, data)
+
+    def _update_values(self, previous_config):
+        data = getattr(previous_config, 'values', []).copy()
+        new_data = getattr(self, 'values', []).copy()
+        for new_item in new_data:
+            added = True
+            for index, item in enumerate(data):
+                if (
+                    item['name'] == new_item['name'] and
+                    item.get('ptype') == new_item.get('ptype') and
+                    item.get('group') == new_item.get('group')
+                ):
+                    data.pop(index)
+                    if not new_item['value']:
+                        added = False
+                    else:  # force to string
+                        new_item['value'] = str(new_item['value'])
+                    break
+            if added:
+                data.append(new_item)
+        setattr(self, 'values', data)
+
+    def _update_values_refs(self, previous_config):
+        data = getattr(previous_config, 'values_refs', {}).copy()
+        new_data = getattr(self, 'values_refs', {}).copy()
+        # remove config keys if a null value is provided
+        for ptype, values in new_data.items():
+            if not values:
+                # error if unsetting non-existing key
+                if ptype not in data:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(ptype, 'values_refs'))
+                data.pop(ptype)
+            else:
+                values_refs = data.get(ptype, [])
+                values_refs.extend(new_data.get(ptype, []))
+                data[ptype] = list(set(values_refs))
+        setattr(self, 'values_refs', data)
 
     def _merge_data(self, field, data, new_data):
         # remove config keys if a null value is provided
@@ -102,18 +207,7 @@ def _merge_data(self, field, data, new_data):
                 data[key] = value
         return data
 
-    def _set_registry(self):
-        # lower case all registry options for consistency
-        self.registry = {key.lower(): value for key, value in self.registry.copy().items()}
-
-        # PORT must be set if private registry is being used
-        if self.registry and self.values.get('PORT', None) is None:
-            # only thing that can get past post_save in the views
-            raise DryccException(
-                'PORT needs to be set in the config '
-                'when using a private registry')
-
-    def _set_tags(self, previous_config):
+    def _update_tags(self, previous_config):
         """verify the tags exist on any nodes as labels"""
         data = getattr(previous_config, 'tags', {}).copy()
         new_data = getattr(self, 'tags', {}).copy()
@@ -141,7 +235,7 @@ def _set_tags(self, previous_config):
                     'tags', data.get(ptype, {}), values)
         setattr(self, 'tags', data)
 
-    def _set_limits(self, previous_config):
+    def _update_limits(self, previous_config):
         data = getattr(previous_config, 'limits', {}).copy()
         new_data = getattr(self, 'limits', {}).copy()
         # check procfile
@@ -152,34 +246,3 @@ def _set_limits(self, previous_config):
                         "the %s has already been used and cannot be deleted" % key)
         self._merge_data('limits', data, new_data)
         setattr(self, 'limits', data)
-
-    def _set_healthcheck(self, previous_config):
-        data = getattr(previous_config, 'healthcheck', {}).copy()
-        new_data = getattr(self, 'healthcheck', {}).copy()
-        # remove config keys if a null value is provided
-        for key, value in new_data.items():
-            if value is None:
-                # error if unsetting non-existing key
-                if key not in data:
-                    raise UnprocessableEntity(
-                        '{} does not exist under {}'.format(key, 'healthcheck'))
-                data.pop(key)
-            else:
-                data[key] = self._merge_data('healthcheck', data.get(key, {}), value)
-        setattr(self, 'healthcheck', data)
-
-    def _set_typed_values(self, previous_config):
-        data = getattr(previous_config, 'typed_values', {}).copy()
-        new_data = getattr(self, 'typed_values', {}).copy()
-        # remove config keys if a null value is provided
-        for ptype, values in new_data.items():
-            if not values:
-                # error if unsetting non-existing key
-                if ptype not in data:
-                    raise UnprocessableEntity(
-                        '{} does not exist under {}'.format(ptype, 'typed_values'))
-                data.pop(ptype)
-            else:
-                data[ptype] = self._merge_data(
-                    'typed_values', data.get(ptype, {}), values)
-        setattr(self, 'typed_values', data)
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index de0cf06b6..aeb0898c1 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -12,9 +12,9 @@
 from scheduler import KubeHTTPException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 
-from ..utils import CacheLock, DeployLock
+from ..utils import DeployLock, dict_diff
 
-from .base import UuidAuditedModel
+from .base import UuidAuditedModel, PTYPE_WEB
 from .appsettings import AppSettings
 
 
@@ -58,7 +58,7 @@ def __str__(self):
     def ptypes(self):
         if self.build is not None:
             return self.build.ptypes
-        return []
+        return [PTYPE_WEB]
 
     @property
     def version_name(self):
@@ -159,18 +159,40 @@ def get_port(self, ptype):
         then use default port or read from ENV var, otherwise attempt to pull from
         the container image
         """
-        return int(self.config.typed_values.get(
-            ptype, {}).get(
-                'PORT', self.config.values.get('PORT', DEFAULT_CONTAINER_PORT)))
+        return int(self.config.envs(ptype).get('PORT', DEFAULT_CONTAINER_PORT))
+
+    def diff_ptypes(self, ptypes):
+        def _get_full_deploy(release, ptypes):
+            deploy = {}
+            for ptype in ptypes:
+                deploy[ptype] = {
+                    'image': release.get_deploy_image(ptype),
+                    'args': release.get_deploy_args(ptype),
+                    'command': release.get_deploy_command(ptype),
+                }
+            return deploy
+
+        pre_release = self.previous()
+        if pre_release is None:
+            return ptypes
+        changed_ptypes = set(self.config.diff_ptypes(pre_release.config, ptypes))
+        if pre_release.build and self.build:
+            deploy = _get_full_deploy(self, ptypes)
+            pre_deploy = _get_full_deploy(pre_release, ptypes)
+            for value in dict_diff(deploy, pre_deploy).values():
+                changed_ptypes = changed_ptypes.union(value.keys())
+        return list(changed_ptypes)
 
     def deploy(self, ptypes=None, force_deploy=False):
+        if not self.build:
+            raise DryccException("there are no builds available for deploy")
         ptypes = set(ptypes).intersection(self.ptypes) if ptypes else self.ptypes
         if not ptypes:
-            self.log(f'skip deploy, ptypes are not within the optional range: {self.ptypes}')
-            return
+            raise DryccException(
+                f'skip deploy, ptypes are not within the optional range: {self.ptypes}')
         # change deployed_ptypes lock
         msg = 'there is an executing pipeline, please wait or force deploy'
-        lock = CacheLock("release:%s" % self.pk)
+        lock = self.app.lock()
         try:
             if lock.acquire() or force_deploy:
                 deployed_ptypes = list(set(self.deployed_ptypes).union(ptypes))
@@ -180,13 +202,16 @@ def deploy(self, ptypes=None, force_deploy=False):
                 raise DryccException(msg)
         finally:
             lock.release()
+        # diff ptypes
+        if not force_deploy and self.app.appsettings_set.latest().autodeploy:
+            ptypes = self.diff_ptypes(ptypes)
         # deploy lock
         lock = DeployLock(self.app.pk)
         if not lock.acquire(ptypes, force=force_deploy):
             raise DryccException(f"{msg}: {lock.locked(ptypes)}")
         run_pipeline.delay(self, ptypes, force_deploy)
 
-    def previous(self):
+    def previous(self, state="succeed"):
         """
         Return the previous Release to this one.
 
@@ -195,8 +220,7 @@ def previous(self):
         releases = self.app.release_set
         if self.pk:
             releases = releases.exclude(pk=self.pk)
-
-        q = Q(failed=False, state="succeed")
+        q = Q(failed=False, state="succeed") if state else Q(failed=False)
         try:
             app_settings = self.app.appsettings_set.latest()
             if app_settings.autorollback is False:
@@ -249,7 +273,7 @@ def rollback(self, user, ptypes=None, version=None):
                 new_release.save(update_fields=["state", "failed", "summary", "exception"])
             raise DryccException(str(e)) from e
 
-    def cleanup_old(self, ptypes=None):
+    def clean(self, ptypes=None):
         """
         Cleanup any old resources from Kubernetes
 
@@ -373,13 +397,14 @@ def _delete_release_in_scheduler(self, namespace, ptypes, version_name):
         }
         if ptypes is not None:
             labels['type__in'] = ptypes
-        # see if the app config has deploy timeout preference, otherwise use global
-        timeout = self.config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)
-
         replica_sets = self.scheduler().rs.get(namespace, labels=labels).json()['items']
         if not replica_sets:
             replica_sets = []
         for replica_set in replica_sets:
+            # see if the app config has deploy timeout preference, otherwise use global
+            ptype = replica_set['metadata'].get("labels", {}).get("type", PTYPE_WEB)
+            timeout = self.config.envs(ptype).get(
+                'DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)
             # Deployment takes care of this in the API, RS does not
             # Have the RS scale down pods and delete itself
             try:
@@ -408,7 +433,7 @@ def save(self, *args, **kwargs):  # noqa
                 for field, diff in self.config.diff(old_config).items():
                     diff_list = []
                     for diff_type, values in diff.items():
-                        diff_list.append(f'{diff_type} {field} {", ".join(values.keys())}')
+                        diff_list.append(f'{diff_type} {field} {", ".join(values)}')
                     if diff_list:
                         changes = ', '.join(diff_list)
                         self.summary += "{} {}".format(self.config.owner, changes)
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 5d53c7717..b47920b8c 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -20,6 +20,7 @@
 from api.models.base import PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 from .schemas import rules
+from .schemas.config import SCHEMA as CONFIG_SCHEMA
 from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
 from .schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
 from .schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
@@ -259,7 +260,6 @@ class ConfigSerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     owner = serializers.ReadOnlyField(source='owner.username')
     values = JSONFieldSerializer(required=False, binary=True)
-    typed_values = JSONFieldSerializer(required=False, binary=True)
     limits = JSONFieldSerializer(required=False, binary=True)
     lifecycle_post_start = JSONFieldSerializer(required=False, binary=True)
     lifecycle_pre_stop = JSONFieldSerializer(required=False, binary=True)
@@ -276,30 +276,25 @@ class Meta:
 
     @staticmethod
     def validate_values(data):
-        for key, value in data.items():
-            if not re.match(CONFIGKEY_MATCH, key):
+        validate_json(data, CONFIG_SCHEMA, serializers.ValidationError)
+        for item in data:
+            if (
+                not re.match(CONFIGKEY_MATCH, item['name']) or
+                (item.get('group') and not re.match(CONFIGKEY_MATCH, item['group']))
+            ):
                 raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
-            if value is None:  # use NoneType to unset an item
+            if item['value'] is None:  # use NoneType to unset an item
                 continue
             # Validate PORT
-            if key == 'PORT':
-                if not str(value).isnumeric():
+            if item['name'] == 'PORT':
+                if not str(item['value']).isnumeric():
                     raise serializers.ValidationError('PORT can only be a numeric value')
-                elif int(value) not in range(1, 65536):
+                elif int(item['value']) not in range(1, 65536):
                     # check if hte port is between 1 and 65535. One extra added for range()
                     # http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_serviceport
                     raise serializers.ValidationError('PORT needs to be between 1 and 65535')
         return data
 
-    @classmethod
-    def validate_typed_values(cls, data):
-        for ptype, values in data.items():
-            validate_ptype(ptype)
-            if values is None:  # use NoneType to unset an item
-                continue
-            cls.validate_values(values)
-        return data
-
     @staticmethod
     def validate_limits(data):
         req_plan_ids = []
@@ -363,15 +358,28 @@ def validate_tags(data):
                         "Tag values must be alphanumeric or \"-_.\", and 1-63 characters.")
         return data
 
+    @staticmethod
+    def validate_values_refs(data):
+        for ptype, values in data.items():
+            validate_ptype(ptype)
+            for value in values:
+                if value is None:
+                    continue
+                if not re.match(CONFIGKEY_MATCH, value):
+                    raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
+        return data
+
     @staticmethod
     def validate_registry(data):
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
+        for ptype, values in data.items():
+            validate_ptype(ptype)
+            if not values:
                 continue
-
-            if not re.match(CONFIGKEY_MATCH, key):
-                raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
-
+            for key, value in values.items():
+                if value is None:
+                    continue
+                if not re.match(CONFIGKEY_MATCH, key):
+                    raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
         return data
 
     @staticmethod
diff --git a/rootfs/api/serializers/schemas/config.py b/rootfs/api/serializers/schemas/config.py
new file mode 100644
index 000000000..4b15877d5
--- /dev/null
+++ b/rootfs/api/serializers/schemas/config.py
@@ -0,0 +1,30 @@
+SCHEMA = {
+    "description": "Config values are a list of config item.",
+    "$schema": "http://json-schema.org/schema#",
+    "items": {
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "ptype": {"type": "string"},
+                    "value": {"oneOf": [{"type": "string"}, {"type": "null"}]},
+                },
+                "required": ["name", "ptype", "value"],
+                "additionalProperties": False,
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "group": {"type": "string"},
+                    "value": {"oneOf": [{"type": "string"}, {"type": "null"}]},
+                },
+                "required": ["name", "group", "value"],
+                "additionalProperties": False,
+            },
+        ]
+    },
+    "minItems": 0,
+    "type": "array"
+}
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index f306a57d5..50ab9699f 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -308,29 +308,6 @@
 DRYCC_FILER_DURATION = int(os.environ.get('DRYCC_FILER_DURATION', '3600'))
 DRYCC_FILER_WAITTIME = int(os.environ.get('DRYCC_FILER_WAITTIME', '1200'))
 
-# True, true, yes, y and more evaluate to True
-# False, false, no, n and more evaluate to False
-# see the above for all available options
-#
-# If a user deploys one build with a Procfile but then forgets to in the next one
-# then let that go through without scaling the missing process types down
-#
-# If the user has a Procfile in both deploys then processes are scaled up / down as per usual
-#
-# By default the process types are scaled down unless this setting is turned on
-DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE = os.environ.get(
-    'DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE', 'true').lower() == "true"
-
-# True, true, yes, y and more evaluate to True
-# False, false, no, n and more evaluate to False
-# see the above for all available options
-#
-# If a previous deploy had a Procfile but then the following deploy has no Procfile then it will
-# result in a 406 - Not Acceptable
-# Has priority over DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE
-DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING = os.environ.get(
-    'DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING', 'false').lower() == "true"
-
 # Define a global default on how many pods to bring up and then
 # take down sequentially during a deploy
 # Defaults to None, the default is to deploy to as many nodes as
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 922262891..370f0537a 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -324,8 +324,12 @@ def test_app_transfer(self, mock_requests):
         # pretend the owner and a collaborator added some config to the app to ensure
         # resources owned by the owner are transferred, but not resources owned by the
         # collaborator.
-        config1 = Config.objects.create(owner=owner, app=app, values={'FOO': 'bar'})
-        config2 = Config.objects.create(owner=collaborator, app=app, values={'CAR': 'star'})
+
+        config1 = Config.objects.create(
+            owner=owner, app=app, values=[{"name": "FOO", "value": "bar", "group": "global"}])
+        config2 = Config.objects.create(
+            owner=collaborator, app=app,
+            values=[{"name": "CAR", "value": "star", "group": "global"}])
 
         # Transfer App
         url = '/v2/apps/{}'.format(app.id)
@@ -630,11 +634,15 @@ def test_gather_app_settings(self, mock_requests):
         Config.objects.create(
             owner=self.user,
             app=app,
-            values={
-                'DRYCC_DEPLOY_TIMEOUT': '60',
-                'DRYCC_DEPLOY_BATCHES': '3',
-                'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS': '60'
-            })
+            values=[
+                {"name": "DRYCC_DEPLOY_TIMEOUT", "value": "60", "group": "global"},
+                {"name": "DRYCC_DEPLOY_BATCHES", "value": "3", "group": "global"},
+                {
+                    "name": "KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS",
+                    "value": "60", "group": "global"
+                },
+            ]
+        )
         s = app._gather_app_settings(app.release_set.latest(),
                                      app.appsettings_set.latest(),
                                      'web',
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 60c5a5c7b..a7ba312b1 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -7,7 +7,6 @@
 
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.test.utils import override_settings
 from unittest import mock
 
 from api.models.build import Build
@@ -212,7 +211,6 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-    @override_settings(DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE=True)
     def test_build_forgotten_procfile(self, mock_requests):
         """
         Test that when a user first posts a build with a Procfile
@@ -271,7 +269,18 @@ def test_build_forgotten_procfile(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.json()['structure'], {'web': 1})
 
-    @override_settings(DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE=False)
+        # do another deploy for this time forget Procfile
+        url = f"/v2/apps/{app_id}/build"
+        body = {'image': 'autotest/example:v3', 'stack': 'container'}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.json()['structure'], {'web': 1})
+
     def test_build_no_remove_process(self, mock_requests):
         """
         Specifically test PROCFILE_REMOVE_PROCS_ON_DEPLOY being turned off
@@ -280,6 +289,9 @@ def test_build_no_remove_process(self, mock_requests):
         """
         # start with a new app
         app_id = self.create_app()
+        # Set routable to false
+        response = self.client.post(f'/v2/apps/{app_id}/settings', {'autodeploy': False})
+        self.assertEqual(response.status_code, 201, response.data)
 
         # post a new build with procfile
         url = f"/v2/apps/{app_id}/build"
@@ -289,15 +301,20 @@ def test_build_no_remove_process(self, mock_requests):
             'stack': 'heroku-18',
             'procfile': {
                 'web': 'node server.js',
+                'task': 'node task.js',
                 'worker': 'node worker.js'
             }
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
+        # deploy
+        response = self.client.post(f"/v2/apps/{app_id}/releases/deploy/")
+        self.assertEqual(response.status_code, 204, response.data)
+
         # scale worker
         url = f"/v2/apps/{app_id}/ptypes/scale"
-        body = {'worker': 1}
+        body = {'worker': 1, 'task': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
@@ -305,73 +322,70 @@ def test_build_no_remove_process(self, mock_requests):
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
+        self.assertPodContains(response.data['results'], app_id, 'task', "v2", "up")
         self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
         url = f"/v2/apps/{app_id}/build"
-        body = {'image': 'autotest/example', 'stack': 'container'}
+        body = {'image': 'autotest/example:v10', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
+        # deploy
+        response = self.client.post(f"/v2/apps/{app_id}/releases/deploy/")
+        self.assertEqual(response.status_code, 204, response.data)
+
         # verify web
         url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual([item for item in response.data['results'] if item["type"] != "web"], [])
+        self.assertEqual(
+            set([item["type"] for item in response.data['results']]),
+            set(["web", "task", "worker"]),
+        )
         self.assertPodContains(response.data['results'], app_id, 'web', "v3", "up")
 
         # look at the app structure
         url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.json()['structure'], {'web': 1})
+        self.assertEqual(response.json()['structure'], {'web': 1, 'task': 1, 'worker': 1})
 
-    @override_settings(DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING=True)
-    def test_build_forgotten_procfile_reject(self, mock_requests):
-        """
-        Test that when a user first posts a build with a Procfile
-        and then later without it that missing process are actually
-        scaled down
-        """
-        # start with a new app
-        app_id = self.create_app()
+        # clean worker
+        response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id), {"ptypes": "task"})
+        self.assertEqual(response.status_code, 204, response.data)
 
-        # post a new build with procfile
-        url = f"/v2/apps/{app_id}/build"
-        body = {
-            'image': 'autotest/example',
-            'stack': 'heroku-18',
-            'sha': 'a'*40,
-            'procfile': {
-                'web': 'node server.js',
-                'worker': 'node worker.js'
-            }
-        }
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.json()['structure'], {'web': 1, 'worker': 1})
 
-        # scale worker
-        url = f"/v2/apps/{app_id}/ptypes/scale"
-        body = {'worker': 1}
-        response = self.client.post(url, body)
+        # clean all worker
+        response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id))
         self.assertEqual(response.status_code, 204, response.data)
 
-        # verify worker
-        url = f"/v2/apps/{app_id}/pods/"
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
         # do another deploy for this time forget Procfile
         url = f"/v2/apps/{app_id}/build"
-        body = {'image': 'autotest/example', 'stack': 'container'}
+        body = {'image': 'autotest/example:v11', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        # verify web
-        url = f"/v2/apps/{app_id}/pods/"
+
+        # deploy
+        response = self.client.post(f"/v2/apps/{app_id}/releases/deploy/")
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertPodContains(response.data['results'], app_id, 'web', "v3", "up")
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
     def test_build_str(self, mock_requests):
         """Test the text representation of a build."""
@@ -513,13 +527,13 @@ def test_build_image_in_registry_with_auth(self, mock_requests):
 
         # add the required PORT information
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'PORT': '80'})}
+        body = {'values': [{"name": "PORT", "group": "global", "value": "80"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # set some registry information
         url = f'/v2/apps/{app_id}/config'
-        body = {'registry': json.dumps({'username': 'bob', 'password': 'zoomzoom'})}
+        body = {'registry': {'web': {'username': 'bob', 'password': 'zoomzoom'}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -535,9 +549,9 @@ def test_build_image_in_registry_with_auth_no_port(self, mock_requests):
 
         # set some registry information
         url = f'/v2/apps/{app_id}/config'
-        body = {'registry': json.dumps({'username': 'bob', 'password': 'zoomzoom'})}
+        body = {'registry': json.dumps({'web': {'username': 'bob', 'password': 'zoomzoom'}})}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(response.status_code, 201, response.data)
 
     def test_release_create_failure(self, mock_requests):
         """
@@ -591,9 +605,8 @@ def test_build_failures(self, mock_requests):
         # create a failed build to check that failed release is created
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
-
             url = f"/v2/apps/{app_id}/build"
-            body = {'image': 'autotest/example', 'stack': 'container'}
+            body = {'image': 'autotest/example:v100', 'stack': 'container'}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
             data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
@@ -601,8 +614,7 @@ def test_build_failures(self, mock_requests):
 
         # create a config to see that the new release is created with the last successful build
         url = f"/v2/apps/{app_id}/config"
-
-        body = {'values': json.dumps({'Test': 'test'})}
+        body = {'values': [{"name": "Test", "group": "global", "value": "test"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(app.release_set.latest().version, 4)
@@ -795,6 +807,7 @@ def test_dryccfile_format(self, mock_requests):
             }
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
+
             body['dryccfile']['run'] = [
                 {
                     'command': ["bash", "-c"],
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 42834edd7..a02613917 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -4,7 +4,6 @@
 
 Run the tests with "./manage.py test api"
 """
-import json
 from io import StringIO
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
@@ -57,75 +56,188 @@ def test_config(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('values', response.data)
-        self.assertEqual(response.data['values'], {})
+        self.assertEqual(response.data['values'], [])
         config1 = response.data
 
         # set an initial config value
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        value1 = {"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         config2 = response.data
         self.assertNotEqual(config1['uuid'], config2['uuid'])
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
 
         # read the config
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         config3 = response.data
         self.assertEqual(config2, config3)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
 
         # set an additional config value
-        body = {'values': json.dumps({'NEW_URL2': 'http://localhost:8080/'})}
+        value2 = {"name": "NEW_URL2", "value": "http://localhost:8080/", "group": "global"}
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         config3 = response.data
         self.assertNotEqual(config2['uuid'], config3['uuid'])
-        self.assertIn('NEW_URL1', response.data['values'])
-        self.assertIn('NEW_URL2', response.data['values'])
+        self.assertEqual([value1, value2], response.data['values'])
 
         # read the config again
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         config4 = response.data
         self.assertEqual(config3, config4)
-        self.assertIn('NEW_URL1', response.data['values'])
-        self.assertIn('NEW_URL2', response.data['values'])
+        self.assertEqual([value1, value2], response.data['values'])
 
         # unset a config value
-        body = {'values': json.dumps({'NEW_URL2': None})}
+        value2['value'] = None
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         config5 = response.data
         self.assertNotEqual(config4['uuid'], config5['uuid'])
-        self.assertNotIn('NEW_URL2', json.dumps(response.data['values']))
+        self.assertEqual([value1], response.data['values'])
 
         # unset all config values
-        body = {'values': json.dumps({'NEW_URL1': None})}
+        value1['value'] = None
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertNotIn('NEW_URL1', json.dumps(response.data['values']))
+        self.assertEqual([], response.data['values'])
 
         # set a port and then unset it to make sure validation ignores the unset
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value3 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value3]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value3], response.data['values'])
 
-        body = {'values': json.dumps({'PORT': None})}
+        value3['value'] = None
+        body = {'values': [value3]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertNotIn('PORT', response.data['values'])
+        self.assertEqual([], response.data['values'])
 
-        # disallow put/patch/delete
+        # disallow put/patch
         response = self.client.put(url)
         self.assertEqual(response.status_code, 405, response.data)
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
         return config5
 
+    def test_registry_set(self, mock_requests):
+        app_id = self.create_app()
+        # set an initial config value
+        url = f"/v2/apps/{app_id}/config"
+        web_registry = {'web': {'username': 'admin', 'password': 'admin'}}
+        body = {'registry': web_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data['registry'], web_registry, response.data)
+
+        task_registry = {'task': {'username': 'admin', 'password': 'admin'}}
+        body = {'registry': task_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        all_registry = {}
+        all_registry.update(web_registry)
+        all_registry.update(task_registry)
+        self.assertEqual(response.data['registry'], all_registry, response.data)
+
+        # delete task username
+        task_registry = {'task': {'username': None, 'password': 'admin'}}
+        body = {'registry': task_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        all_registry = {}
+        all_registry.update(web_registry)
+        all_registry.update(task_registry)
+        del all_registry['task']['username']
+        self.assertEqual(response.data['registry'], all_registry, response.data)
+
+        # delete task registry
+        task_registry = {'task': None}
+        body = {'registry': task_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data['registry'], web_registry, response.data)
+
+    def test_values_refs(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+        values = [{"name": "DEBUG", "value": "true", "group": "global"}]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'true'})
+        self.assertEqual(config.envs("task"), {'DEBUG': 'true'})
+
+        # add test error, changed nothing
+        values = [
+            {"name": "DEBUG", "value": "true", "group": "global"},
+        ]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 409, response.data)
+
+        # add values ok
+        values = [
+            {"name": "DEBUG", "value": "true", "group": "mytask1"},
+            {"name": "APP", "value": "task2", "group": "mytask2"},
+            {"name": "DEBUG", "value": "false", "group": "global"},
+        ]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'false'})
+        self.assertEqual(config.envs("task"), {'DEBUG': 'false'})
+
+        # add values_refs
+        body = {'values_refs': {"task": ["mytask1", "mytask2"]}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'false'})
+        self.assertEqual(config.envs("task"), {'DEBUG': 'true', 'APP': 'task2'})
+
+        # add new item for mytask
+        values = [
+            {"name": "VERSION", "value": "1.0.1", "group": "mytask1"},
+        ]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'false'})
+        self.assertEqual(
+            config.envs("task"), {'DEBUG': 'true', 'VERSION': '1.0.1', 'APP': 'task2'})
+
+        # remove tas values_refs
+        body = {'values_refs': {'task': ['mytask1']}}
+        response = self.client.delete(url, body)
+        self.assertEqual(response.status_code, 200, response.data)
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data['values_refs'], {"task": ["mytask2"]}, response.data)
+
     def test_response_data(self, mock_requests):
         """Test that the serialized response contains only relevant data."""
         app_id = self.create_app()
@@ -133,17 +245,18 @@ def test_response_data(self, mock_requests):
         url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         for key in response.data:
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values',
-                                'typed_values', 'limits', 'tags', 'registry', 'healthcheck',
+                                'values_refs', 'limits', 'tags', 'registry', 'healthcheck',
                                 'lifecycle_post_start', 'lifecycle_pre_stop',
                                 'termination_grace_period'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
-            'values': {'PORT': '5000'},
+            'values': [value1],
             'limits': {
                 PTYPE_RUN: 'std1.large.c1m1',
                 PTYPE_WEB: 'std1.large.c1m1'
@@ -158,9 +271,9 @@ def test_response_data_types_converted(self, mock_requests):
         app_id = self.create_app()
 
         url = f"/v2/apps/{app_id}/config"
-
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
         body = {
-            'values': json.dumps({'PORT': 5000}),
+            'values': [value1],
             'limits': {
                 PTYPE_WEB: 'std1.large.c1m2',
             }
@@ -169,13 +282,13 @@ def test_response_data_types_converted(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         for key in response.data:
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'limits',
-                                'typed_values', 'tags', 'registry', 'healthcheck',
+                                'values_refs', 'tags', 'registry', 'healthcheck',
                                 'lifecycle_post_start', 'lifecycle_pre_stop',
                                 'termination_grace_period'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
-            'values': {'PORT': '5000'},
+            'values': [value1],
             'limits': {
                 PTYPE_RUN: 'std1.large.c1m1',
                 PTYPE_WEB: 'std1.large.c1m2'
@@ -201,17 +314,18 @@ def test_config_set_same_key(self, mock_requests):
         url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
 
         # reset same config value
-        body = {'values': json.dumps({'PORT': '5001'})}
+        value1['value'] = '5001'
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
-        self.assertEqual(response.data['values']['PORT'], '5001')
+        self.assertEqual([value1], response.data['values'])
 
     def test_config_set_unicode(self, mock_requests):
         """
@@ -221,23 +335,24 @@ def test_config_set_unicode(self, mock_requests):
         url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': json.dumps({'POWERED_BY': 'Деис'})}
+        value1 = {"name": "POWERED_BY", "value": "Деис", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('POWERED_BY', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         # reset same config value
-        body = {'values': json.dumps({'POWERED_BY': 'Кроликов'})}
+        value1['value'] = 'Кроликов'
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('POWERED_BY', response.data['values'])
-        self.assertEqual(response.data['values']['POWERED_BY'], 'Кроликов')
+        self.assertEqual([value1], response.data['values'])
 
         # set an integer to test unicode regression
-        body = {'values': json.dumps({'INTEGER': 1})}
+        value2 = {"name": "INTEGER", "value": "1", "group": "global"}
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('INTEGER', response.data['values'])
-        self.assertEqual(response.data['values']['INTEGER'], '1')
+        self.assertEqual([value1, value2], response.data['values'])
 
     def test_config_str(self, mock_requests):
         """Test the text representation of a node."""
@@ -251,11 +366,14 @@ def test_valid_config_keys(self, mock_requests):
         keys = ("FOO", "_foo", "f001", "FOO_BAR_BAZ_")
         app_id = self.create_app()
         url = f'/v2/apps/{app_id}/config'
+        values = []
         for k in keys:
-            body = {'values': json.dumps({k: "testvalue"})}
+            value1 = {"name": k, "value": "testvalue", "group": "global"}
+            values.append(value1)
+            body = {'values': [value1]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201)
-            self.assertIn(k, response.data['values'])
+            self.assertEqual(values, response.data['values'])
 
     def test_config_deploy_failure(self, mock_requests):
         """
@@ -273,7 +391,8 @@ def test_config_deploy_failure(self, mock_requests):
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
             url = f'/v2/apps/{app_id}/config'
-            body = {'values': json.dumps({'test': "testvalue"})}
+            value1 = {"name": "test", "value": "testvalue", "group": "global"}
+            body = {'values': [value1]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201)
             app = App.objects.get(id=app_id)
@@ -288,7 +407,8 @@ def test_invalid_config_keys(self, mock_requests):
         app_id = self.create_app()
         url = f'/v2/apps/{app_id}/config'
         for k in keys:
-            body = {'values': json.dumps({k: "testvalue"})}
+            value1 = {"name": k, "value": "testvalue", "group": "global"}
+            body = {'values': [value1]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400)
 
@@ -298,13 +418,13 @@ def test_invalid_config_values(self, mock_requests):
         Right now only PORT is checked
         """
         data = [
-            {'field': 'PORT', 'value': 'dog'},
-            {'field': 'PORT', 'value': 99999}
+            {"name": "PORT", "value": "dog", "group": "global"},
+            {"name": "PORT", "value": "99999", "group": "global"}
         ]
         app_id = self.create_app()
         url = f'/v2/apps/{app_id}/config'
         for row in data:
-            body = {'values': json.dumps({row['field']: row['value']})}
+            body = {'values': [row]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
 
@@ -321,10 +441,11 @@ def test_admin_can_create_config_on_other_apps(self, mock_requests):
 
         # set an initial config value
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         return response
 
     def test_config_owner_is_requesting_user(self, mock_requests):
@@ -348,7 +469,8 @@ def test_unauthorized_user_cannot_modify_config(self, mock_requests):
         unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
         url = '/v2/apps/{}/config'.format(app_id)
-        body = {'values': {'FOO': 'bar'}}
+        value1 = {"name": "FOO", "value": "bar", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
 
@@ -371,17 +493,19 @@ def test_config_failures(self, mock_requests):
 
         # set an initial config value
         url = f"/v2/apps/{app_id}/config"
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        value1 = {"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         success_config = app.release_set.latest().config
 
         # create a failed config to check that failed release is created
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
             url = f'/v2/apps/{app_id}/config'
-            body = {'values': json.dumps({'test': "testvalue"})}
+            value1 = {"name": "test", "value": "testvalue", "group": "global"}
+            body = {'values': [value1]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201)
             self.assertEqual(app.release_set.latest().version, 4)
@@ -401,8 +525,9 @@ def test_config_failures(self, mock_requests):
     def test_unset_limits(self, mock_requests):
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
         body = {
-            'values': json.dumps({'PORT': 5000}),
+            'values': [value1],
             'limits': {
                 "task": 'std1.large.c2m4',
                 PTYPE_RUN: 'std1.large.c2m4',
@@ -431,8 +556,9 @@ def test_unset_limits(self, mock_requests):
     def test_unset_limits_error(self, mock_requests):
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
         body = {
-            'values': json.dumps({'PORT': 5000}),
+            'values': [value1],
             'limits': {
                 "task": 'std1.large.c2m4',
                 PTYPE_RUN: 'std1.large.c2m4',
@@ -504,8 +630,9 @@ def test_measure_config(self, *args, **kwargs):
         # create
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
         body = {
-            'values': json.dumps({'PORT': 5000}),
+            'values': [value1],
             'limits': {
                   PTYPE_RUN: 'std1.large.c2m4',
                   PTYPE_WEB: 'std1.large.c2m4',
@@ -543,8 +670,9 @@ def test_set_config_limits_run(self, *args, **kwargs):
         )
         # deploy
         release.deploy()
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
         body = {
-            'values': json.dumps({'PORT': 5000}),
+            'values': [value1],
             'limits': {
                   PTYPE_RUN: 'std1.large.c2m4',
                   PTYPE_WEB: 'std1.large.c2m4',
@@ -566,27 +694,21 @@ def test_config_set_typed_values(self, mock_requests):
         url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'typed_values': {'web': {'PORT': '5000'}}}
+        body = {'values': [{"name": "PORT", "value": "5000", "ptype": "web"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['typed_values']['web'])
+        self.assertIn({"name": "PORT", "value": "5000", "ptype": "web"}, response.data['values'])
 
         # reset same config value
-        body = {'typed_values': {'web': {'PORT': '5001'}}}
+        body = {'values': [{"name": "PORT", "value": "5001", "ptype": "web"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['typed_values']['web'])
-        self.assertEqual(response.data['typed_values']['web']['PORT'], '5001')
+        self.assertIn({"name": "PORT", "value": "5001", "ptype": "web"}, response.data['values'])
         # unset PORT
-        body = {'typed_values': {'web': {'PORT': None}}}
+        body = {'values': [{"name": "PORT", "value": None, "ptype": "web"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['typed_values']['web'], {}, response.data)
-        # unset web
-        body = {'typed_values': {'web': None}}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['typed_values'], {}, response.data)
+        self.assertEqual(response.data['values'], [], response.data)
 
     def test_config_version(self, mock_requests):
         """
@@ -596,35 +718,173 @@ def test_config_version(self, mock_requests):
         url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': {'PORT': '5000'}}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         url = f"/v2/apps/{app_id}/config"
 
         # set config NAME
-        body = {'values': {'NAME': 'drycc'}}
+        value2 = {"name": "NAME", "value": "drycc", "group": "global"}
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NAME', response.data['values'])
+        self.assertEqual([value1, value2], response.data['values'])
 
         # set config WEBSITE
-        body = {'values': {'WEBSITE': 'www.drycc.cc'}}
+        value3 = {"name": "WEBSITE", "value": "www.drycc.cc", "group": "global"}
+        body = {'values': [value3]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('WEBSITE', response.data['values'])
+        self.assertEqual([value1, value2, value3], response.data['values'])
 
         url = f"/v2/apps/{app_id}/config/?version=v2"
         response = self.client.get(url)
-        self.assertEqual(response.data['values'], {'PORT': '5000'}, response.data)
+        self.assertEqual(response.data['values'], [value1], response.data)
 
         url = f"/v2/apps/{app_id}/config/?version=v3"
         response = self.client.get(url)
         self.assertEqual(
-            response.data['values'], {'PORT': '5000', 'NAME': 'drycc'}, response.data)
+            response.data['values'], [value1, value2], response.data)
 
         url = f"/v2/apps/{app_id}/config/?version=v4"
         response = self.client.get(url)
         self.assertEqual(
             response.data['values'],
-            {'PORT': '5000', 'NAME': 'drycc', 'WEBSITE': 'www.drycc.cc'}, response.data)
+            [value1, value2, value3], response.data)
+
+    def test_diff_ptypes(self, mock_requests):
+        app_id = self.create_app()
+        value1 = {"name": "WEBSITE", "value": "www.drycc.cc", "group": "global"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), {"web", "task"})
+
+        # add value
+        value1 = {"name": "DEBUG", "value": "true", "group": "mygroup"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), set())
+
+        # add group ref
+        body = {'values_refs': {"task": ["mygroup"]}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), {"task"})
+        # add new group
+        value1 = {"name": "DEBUG", "value": "true", "group": "mygroup1"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        body = {'values_refs': {"task": ["mygroup1"]}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), {"task"})
+        self.assertEqual(config.envs("task"), {'WEBSITE': 'www.drycc.cc', 'DEBUG': 'true'})
+
+    def test_config_from_dryccfile(self, mock_requests):
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        build_body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'dryccfile': {
+                "build": {
+                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
+                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                },
+                'deploy': {
+                    'web': {
+                        'image': "127.0.0.1/cat/cat"
+                    }
+                }
+            },
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+        value1 = {"name": "WEBSITE", "value": "www.drycc.cc", "group": "global"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
+        # set env by dryccfile
+        build_body['dryccfile']['config'] = {
+            "mygroup1": [
+                {"name": "GROUP", "value": "g1"},
+                {"name": "DEBUG", "value": "tr"},
+            ],
+            "mygroup2": [
+                {"name": "TEST1", "value": "g1"},
+                {"name": "TEST2", "value": "tr"},
+            ],
+        }
+        build_body['dryccfile']['deploy']['web']['config'] = {
+            'env': [
+                {'name': "PENV1", 'value': 'web'},
+                {'name': "PENV2", 'value': 'web'},
+            ],
+            'ref': ['mygroup1', 'mygroup2']
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(
+            release.config.envs("web"),
+            {
+                'GROUP': 'g1', 'DEBUG': 'tr', 'TEST1': 'g1', 'TEST2': 'tr', 'PENV1': 'web',
+                'PENV2': 'web'
+            }
+        )
+        build_body['dryccfile']['deploy']['web']['healthcheck'] = {
+            'livenessProbe': {
+                'httpGet': {
+                    'path': '/healthz',
+                    'port': 8080,
+                },
+                'initialDelaySeconds': 3,
+                'periodSeconds': 3,
+            }
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(
+            release.config.healthcheck['web'],
+            build_body['dryccfile']['deploy']['web']['healthcheck'],
+        )
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index b88592e30..5b8a1266a 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -4,7 +4,6 @@
 Run the tests with "./manage.py test api"
 """
 import os
-import json
 
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
@@ -235,7 +234,7 @@ def test_release(self, mock_requests):
 
         # post new config
         url = f"/v2/apps/{app_id}/config"
-        body = {'values': json.dumps({'KEY': 'value'})}
+        body = {'values': [{'name': 'KEY', 'value': 'value', 'group': 'global'}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
diff --git a/rootfs/api/tests/test_registry.py b/rootfs/api/tests/test_registry.py
index 7a0ccbda3..0be570856 100644
--- a/rootfs/api/tests/test_registry.py
+++ b/rootfs/api/tests/test_registry.py
@@ -38,19 +38,19 @@ def test_registry(self, mock_requests):
         self.assertEqual(response.data['registry'], {})
 
         # set some registry information without PORT
-        body = {'registry': json.dumps({'username': 'bob'})}
+        body = {'registry': {'web': {'username': 'bob'}}}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(response.status_code, 201, response.data)
         registry1 = response.data
 
         # set required PORT
-        body = {'values': json.dumps({'PORT': '80'})}
+        body = {'values': [{"name": "PORT", "value": "80", "group": "global"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry1 = response.data
 
-        # set some registry information
-        body = {'registry': json.dumps({'username': 'bob'})}
+        # no change error
+        body = {'registry': {'web': {'username': 'bobb'}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry1 = response.data
@@ -59,45 +59,45 @@ def test_registry(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('registry', response.data)
-        registry = response.data['registry']
+        registry = response.data['registry']['web']
         self.assertIn('username', registry)
-        self.assertEqual(registry['username'], 'bob')
+        self.assertEqual(registry['username'], 'bobb')
 
         # set an additional value
-        # set them upper case, internally it should translate to lower
-        body = {'registry': json.dumps({'PASSWORD': 's3cur3pw1'})}
+        body = {'registry': {'web': {'password': 's3cur3pw1'}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry2 = response.data
         self.assertNotEqual(registry1['uuid'], registry2['uuid'])
-        registry = response.data['registry']
+        registry = response.data['registry']['web']
         self.assertIn('password', registry)
         self.assertEqual(registry['password'], 's3cur3pw1')
         self.assertIn('username', registry)
-        self.assertEqual(registry['username'], 'bob')
+        self.assertEqual(registry['username'], 'bobb')
 
         # read the registry information again
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         registry3 = response.data
         self.assertEqual(registry2, registry3)
-        registry = response.data['registry']
+        registry = response.data['registry']['web']
         self.assertIn('password', registry)
         self.assertEqual(registry['password'], 's3cur3pw1')
         self.assertIn('username', registry)
-        self.assertEqual(registry['username'], 'bob')
+        self.assertEqual(registry['username'], 'bobb')
 
         # unset a value
-        body = {'registry': json.dumps({'password': None})}
+        body = {'registry': {'web': {'password': None}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry4 = response.data
         self.assertNotEqual(registry3['uuid'], registry4['uuid'])
         self.assertNotIn('password', json.dumps(response.data['registry']))
 
-        # bad registry key values
-        body = {'registry': json.dumps({'pa$$w0rd': 'woop'})}
+        # key error
+        body = {'registry': {'web': {'pa$$w0rd': 'woop'}}}
         response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 400, response.data)
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
@@ -106,8 +106,6 @@ def test_registry(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
 
     def test_registry_deploy(self, mock_requests):
         """
@@ -118,20 +116,24 @@ def test_registry_deploy(self, mock_requests):
         # Set mandatory PORT
         response = self.client.post(
             f'/v2/apps/{app_id}/config',
-            {'values': json.dumps({'PORT': '4999'})}
+            {'values': [{"name": "PORT", "value": "4999", "group": "global"}]}
         )
 
         # Set registry information
-        body = {'registry': json.dumps({
-            'username': 'bob',
-            'password': 's3cur3pw1'
-        })}
+        body = {
+            'registry': {
+                'web': {
+                    'username': 'bob',
+                    'password': 's3cur3pw1'
+                }
+            }
+        }
         response = self.client.post(
             f'/v2/apps/{app_id}/config',
             body
         )
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('username', response.data['registry'])
-        self.assertIn('password', response.data['registry'])
-        self.assertEqual(response.data['registry']['username'], 'bob')
-        self.assertEqual(response.data['registry']['password'], 's3cur3pw1')
+        self.assertIn('username', response.data['registry']['web'])
+        self.assertIn('password', response.data['registry']['web'])
+        self.assertEqual(response.data['registry']['web']['username'], 'bob')
+        self.assertEqual(response.data['registry']['web']['password'], 's3cur3pw1')
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 4c4531893..f63a08255 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -45,10 +45,12 @@ def test_release(self, mock_requests):
         app_id = self.create_app()
         # check that updating config rolls a new release
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
         # check to see that an initial release was created
         url = f'/v2/apps/{app_id}/releases'
         response = self.client.get(url)
@@ -212,7 +214,9 @@ def test_get_image(self, mock_requests):
 
     def test_response_data(self, mock_requests):
         app_id = self.create_app()
-        body = {'values': json.dumps({'NEW_URL': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL", "value": "http://localhost:8080/", "group": "global"}]
+        }
         url = '/v2/apps/{}/config'.format(app_id)
         config_response = self.client.post(url, body)
         url = '/v2/apps/{}/releases/v2'.format(app_id)
@@ -286,7 +290,8 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         # update config to roll another release
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        values = [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        body = {'values': values}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # create another release with a different build
@@ -312,7 +317,7 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(release5.config.values, release3.config.values)
         # double-check to see that the current build and config is the same as v3
         self.assertEqual(release5.get_deploy_image(PTYPE_WEB), 'autotest/example')
-        self.assertEqual(release5.config.values, {'NEW_URL1': 'http://localhost:8080/'})
+        self.assertEqual(release5.config.values, values)
         # try to rollback to v1 and verify that the rollback failed
         # (v1 is an initial release with no build)
         url = f"/v2/apps/{app_id}/releases/rollback/"
@@ -327,7 +332,7 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(Release.objects.count(), 6)
         release6 = Release.objects.get(app=app, version=6)
         self.assertEqual(release6.get_deploy_image(PTYPE_WEB), 'autotest/example')
-        self.assertEqual(release6.config.values, {})
+        self.assertEqual(release6.config.values, [])
 
     def test_release_str(self, mock_requests):
         """Test the text representation of a release."""
@@ -345,7 +350,7 @@ def test_release_summary(self, mock_requests):
         # add config, confirm that config objects are in the summary
         url = f'/v2/apps/{app.id}/config'
         body = {
-            'values': json.dumps({'FOO': 'bar'}),
+            'values': [{"name": "FOO", "value": "bar", "group": "global"}],
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -361,11 +366,13 @@ def test_admin_can_create_release(self, mock_requests):
         app_id = self.create_app()
         # check that updating config rolls a new release
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
         # check to see that an initial release was created
         url = f'/v2/apps/{app_id}/releases'
         response = self.client.get(url)
@@ -389,7 +396,9 @@ def test_unauthorized_user_cannot_modify_release(self, mock_requests):
 
         # update config to roll a new release
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         unauthorized_user = User.objects.get(username='autotest2')
         unauthorized_token = self.get_or_create_token(unauthorized_user)
@@ -415,7 +424,9 @@ def test_release_rollback_failure(self, mock_requests):
 
         # update config to roll a new release
         url = '/v2/apps/{}/config'.format(app_id)
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -431,7 +442,9 @@ def test_release_rollback_failure(self, mock_requests):
 
         # update config to roll a new release
         url = '/v2/apps/{}/config'.format(app_id)
-        body = {'values': json.dumps({'NEW_URL2': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL2", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -488,14 +501,18 @@ def test_release_no_change(self, mock_requests):
 
         # check that updating config rolls a new release
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
 
         # trigger identical release
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 409, response.data)
 
@@ -517,19 +534,26 @@ def test_release_get_port(self, mock_requests):
 
         # switch to a dockerfile app or else it'll automatically default to 5000
         url = f'/v2/apps/{app_id}/build'
-        body = {'image': 'autotest/example', 'stack': 'container'}
+        body = {
+            'image': 'autotest/example',
+            'stack': 'container',
+            'procfile': {
+                'web': 'node server.js',
+                'task': 'node worker.js'
+            },
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'PORT': '8080'})}
+        body = {'values': [{"name": "PORT", "value": "8080", "group": "global"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.get_port('web'), 8080)
 
         url = f'/v2/apps/{app_id}/config'
-        body = {'typed_values': json.dumps({"web": {'PORT': '9000'}})}
+        body = {'values': [{"name": "PORT", "value": "9000", "ptype": "web"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
@@ -538,13 +562,13 @@ def test_release_get_port(self, mock_requests):
         # not web procfile
         self.assertEqual(release.get_port('task'), 8080)
         url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'PORT': '9000'})}
+        body = {'values': [{"name": "PORT", "value": "9000", "group": "global"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.get_port('task'), 9000)
-        # set typed_values port
-        body = {'typed_values': json.dumps({"task": {'PORT': '9001'}})}
+        # set task port
+        body = {'values': [{"name": "PORT", "value": "9001", "ptype": "task"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
@@ -650,7 +674,9 @@ def test_release_external_registry(self, mock_requests):
         app_id = self.create_app()
 
         # set the required port for external registries
-        body = {'values': json.dumps({'PORT': '3000'})}
+        body = {
+            'values': [{"name": "PORT", "value": "3000", "group": "global"}]
+        }
         config_response = self.client.post('/v2/apps/{}/config'.format(app_id), body)
         self.assertEqual(config_response.status_code, 201, config_response.data)
 
@@ -664,7 +690,7 @@ def test_release_external_registry(self, mock_requests):
         self.assertEqual(release.get_deploy_image(PTYPE_WEB), 'test/autotest/example')
 
         url = f'/v2/apps/{app_id}/config'
-        body = {'typed_values': json.dumps({"web": {'PORT': '9000'}})}
+        body = {'values': [{"name": "PORT", "value": "9000", "ptype": "web"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
@@ -727,3 +753,134 @@ def test_run(self, mock_requests):
         runners = release.get_runners(["web"])
         self.assertEqual(len(runners), 1)
         self.assertEqual(runners[0]["timeout"], 3600)
+
+    def test_release_diff_ptypes(self, mock_requests):
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        url = f"/v2/apps/{app_id}/build"
+        run_image = "127.0.0.1:7070/myapp/run:git-123fsa1"
+        web_image = "127.0.0.1:7070/myapp/web:git-123fsa1"
+        worker_image = "127.0.0.1:7070/myapp/worker:git-123fsa1"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'dryccfile': {
+                "build": {
+                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
+                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                },
+                "run": [
+                    {
+                        "command": ["./deployment-tasks.sh"],
+                        "image": run_image,
+                    }
+                ],
+                "deploy": {
+                    "web": {
+                        "command": ["bash", "-c"],
+                        "args": ["bundle exec puma -C config/puma.rb"],
+                        "image": web_image
+                    },
+                    "worker": {
+                        "command": ["bash", "-c"],
+                        "args": ["python myworker.py"],
+                        "image": worker_image
+                    }
+                }
+            }
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["web", "worker"])
+
+        body['dryccfile']['config'] = {
+            'g1': [
+                {
+                    'name': 'G1',
+                    'value': 'g1',
+                },
+            ]
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, [])
+        # change worker env
+        body['dryccfile']['deploy']['worker']['config'] = {
+            'ref': ['g1']
+        }
+        body['dryccfile']['config'] = {
+            'g1': [
+                {
+                    'name': 'G1',
+                    'value': 'g1',
+                },
+            ]
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["worker"])
+        # env cover, only change web
+        body['dryccfile']['config'] = {
+            'global': [
+                {
+                    'name': 'G1',
+                    'value': 'g1',
+                },
+            ]
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["web"])
+        # add a global env
+        body['dryccfile']['config'] = {
+            'global': [
+                {
+                    'name': 'G2',
+                    'value': 'g2',
+                },
+            ]
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(set(ptypes), set(["web", "worker"]))
+
+        # change web env
+        body['dryccfile']['deploy']['web']['config'] = {
+            'env': [
+                {'name': 'ENV1', 'value': 'env1'}
+            ]
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["web"])
diff --git a/rootfs/api/tests/test_tags.py b/rootfs/api/tests/test_tags.py
index b2a9d93ff..dd6e81438 100644
--- a/rootfs/api/tests/test_tags.py
+++ b/rootfs/api/tests/test_tags.py
@@ -124,5 +124,3 @@ def test_tags(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index dade13b81..ddf3cfa6b 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -57,6 +57,10 @@
     re_path(
         r"^apps/(?P<id>{})/ptypes/restart/?$".format(settings.APP_URL_REGEX),
         views.PtypeViewSet.as_view({'post': 'restart'})),
+    # clean old k8s resource
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/clean/?$".format(settings.APP_URL_REGEX),
+        views.PtypeViewSet.as_view({'post': 'clean'})),
     # scale ptype replcas
     re_path(
         r"^apps/(?P<id>{})/ptypes/scale/?$".format(settings.APP_URL_REGEX),
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index 0c0621262..3d594a24b 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -193,7 +193,7 @@ def unit_to_bytes(size):
     return round(_)
 
 
-def validate_json(value, schema, raise_exception=ValidationError):
+def validate_json(value, schema, raise_exception=jsonschema.ValidationError):
     if value is not None:
         try:
             jsonschema.validate(value, schema)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index cf334d0d2..c9ada3774 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -396,20 +396,32 @@ class ConfigViewSet(ReleasableViewSet):
     model = models.config.Config
     serializer_class = serializers.ConfigSerializer
 
+    def delete(self, request, **kwargs):
+        values_refs = self.get_serializer().validate_values_refs(request.data.get('values_refs'))
+        if not values_refs or not values_refs.values():
+            raise DryccException("ptype or groups is required")
+
+        config = self.model(app=self.get_app(), owner=self.request.user, values_refs={})
+        old_values_refs = config.previous().values_refs.copy()
+        for ptype, groups in values_refs.items():
+            for group in old_values_refs.get(ptype, []):
+                if group not in groups:
+                    if ptype not in config.values_refs:
+                        config.values_refs[ptype] = [group]
+                    elif group not in config.values_refs[ptype]:
+                        config.values_refs[ptype].append(group)
+        config.save(ignore_update_fields=["values_refs"])
+
+        self.post_save(config)
+        return Response(status=status.HTTP_200_OK)
+
     def post_save(self, config):
         latest_release = config.app.release_set.filter(failed=False).latest()
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
-            if config.app.appsettings_set.latest().autodeploy:
-                ptypes = set()
-                for field, diff in config.diff().items():
-                    if field in config.ptype_fields:
-                        for value in diff.values():
-                            ptypes.update(value.keys())
-                # allof_fields changed, deploy all.
-                ptypes = list(ptypes) if ptypes else None
-                release.deploy(ptypes, False)
+            if release.build and config.app.appsettings_set.latest().autodeploy:
+                release.deploy(release.ptypes, False)
         except BaseException as e:
             config.delete()
             if isinstance(e, AlreadyExists):
@@ -474,6 +486,14 @@ def restart(self, request, *args, **kwargs):
             restart_app.delay(app, **{"type": ptype})
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    def clean(self, request, *args, **kwargs):
+        app = self.get_app()
+        ptypes = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        ptypes = app.check_ptypes(ptypes)
+        app.clean(ptypes=ptypes)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
     def scale(self, request, **kwargs):
         app = self.get_app()
         scale_app.delay(app, request.user, request.data)
diff --git a/rootfs/scheduler/resources/secret.py b/rootfs/scheduler/resources/secret.py
index ad7d3f1eb..41424322c 100644
--- a/rootfs/scheduler/resources/secret.py
+++ b/rootfs/scheduler/resources/secret.py
@@ -117,10 +117,10 @@ def update(self, namespace, name, data, secret_type='Opaque', labels={}):
 
         return response
 
-    def delete(self, namespace, name):
+    def delete(self, namespace, name, ignore_exception=False):
         url = self.api("/namespaces/{}/secrets/{}", namespace, name)
         response = self.http_delete(url)
-        if self.unhealthy(response.status_code):
+        if not ignore_exception and self.unhealthy(response.status_code):
             raise KubeHTTPException(
                 response,
                 'delete Secret "{}" in Namespace "{}"', name, namespace

From 8f85bc6c84a09270e653b1bf650369cdcdf00eef Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 14 Nov 2024 09:16:28 +0800
Subject: [PATCH 094/200] chore(python): upgrade requirements version

---
 rootfs/requirements.txt | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 942de01e7..30a762c64 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -3,28 +3,28 @@ inotify==0.2.10
 backoff==2.2.1
 django==4.2.16
 channels==4.1.0
-aiohttp==3.10.2
-django-cors-headers==4.3.1
+aiohttp==3.11.0
+django-cors-headers==4.6.0
 django-guardian==2.4.0
 djangorestframework==3.15.2
-docker==7.0.0
-gunicorn==22.0.0
-uvicorn[standard]==0.29.0
+docker==7.1.0
+gunicorn==23.0.0
+uvicorn[standard]==0.32.0
 asgiref==3.8.1
-idna==3.7
-jsonschema==4.21.1
+idna==3.10
+jsonschema==4.23.0
 morph==0.1.5
-packaging==24.0
-pyasn1==0.5.1
-psycopg[binary]==3.1.18
-pyOpenSSL==24.1.0
+packaging==24.2
+pyasn1==0.6.1
+psycopg[binary]==3.2.3
+pyOpenSSL==24.2.1
 ndg-httpsclient==0.5.1
-pytz==2024.1
-requests==2.32.0
+pytz==2024.2
+requests==2.32.3
 requests-toolbelt==1.0.0
-celery==5.5.0b3
-hiredis==2.3.2
+celery==5.5.0rc1
+hiredis==3.0.0
 django_redis==5.4.0
-dj-database-url==2.1.0
-social-auth-app-django==5.4.1
-kubernetes==29.0.0
+dj-database-url==2.3.0
+social-auth-app-django==5.4.2
+kubernetes==31.0.0

From 7233f51cf3c688a1f35f4306515f9532154d7325 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 14 Nov 2024 10:43:13 +0800
Subject: [PATCH 095/200] chore(migrations): add registry migrate

---
 .../0019_migration_config_values_and_typed_values.py     | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py b/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
index e96cbe812..8325cded6 100644
--- a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
+++ b/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
@@ -6,8 +6,8 @@
 
 def migration_values(apps, schema_editor):
     with connection.cursor() as cursor:
-        cursor.execute("SELECT uuid,values,typed_values FROM api_config")
-        for uuid, values, typed_values in cursor:
+        cursor.execute("SELECT uuid,values,typed_values,registry FROM api_config")
+        for uuid, values, typed_values, registry in cursor:
             new_values = []
             config = Config.objects.using('default').get(pk=uuid)
             for name, value in json.loads(values).items():
@@ -23,7 +23,12 @@ def migration_values(apps, schema_editor):
                         "ptype": ptype,
                         "value": value
                     })
+            new_registry, data = {}, json.loads(registry)
+            if data:
+                for ptype in config.app.structure.keys():
+                    new_registry[ptype] = data
             config.values = new_values
+            config.registry = new_registry
             config.save(ignore_update_fields=config.allof_fields)
 
 

From 3b4d5e3bba683d86ce9ab0742ced3c3bc142d6bd Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 15 Nov 2024 09:18:11 +0800
Subject: [PATCH 096/200] fix(migrate): master-slave database error

---
 rootfs/api/migrations/0017_migration_deployed_ptypes.py       | 2 +-
 .../0019_migration_config_values_and_typed_values.py          | 2 +-
 rootfs/api/routers.py                                         | 4 ++++
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/migrations/0017_migration_deployed_ptypes.py b/rootfs/api/migrations/0017_migration_deployed_ptypes.py
index 197f82d7c..25d2f7831 100644
--- a/rootfs/api/migrations/0017_migration_deployed_ptypes.py
+++ b/rootfs/api/migrations/0017_migration_deployed_ptypes.py
@@ -5,7 +5,7 @@
 
 
 def migration_deployed_ptypes(apps, schema_editor):
-    for release in Release.objects.all().using("default"):
+    for release in Release.objects.all():
         deployed_ptypes = set()
         for condition in release.conditions:
             if condition["ptypes"] is None:
diff --git a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py b/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
index 8325cded6..c6eab9215 100644
--- a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
+++ b/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
@@ -9,7 +9,7 @@ def migration_values(apps, schema_editor):
         cursor.execute("SELECT uuid,values,typed_values,registry FROM api_config")
         for uuid, values, typed_values, registry in cursor:
             new_values = []
-            config = Config.objects.using('default').get(pk=uuid)
+            config = Config.objects.get(pk=uuid)
             for name, value in json.loads(values).items():
                 new_values.append({
                     "name": name,
diff --git a/rootfs/api/routers.py b/rootfs/api/routers.py
index 801d6c95c..3e5e98115 100644
--- a/rootfs/api/routers.py
+++ b/rootfs/api/routers.py
@@ -32,4 +32,8 @@ def allow_relation(self, obj1, obj2, **hints):
         return True
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
+        if 'replica' in settings.DATABASES and 'model' in hints:
+            model = hints['model']
+            tracker_key = ".".join([model.__module__, model.__name__])
+            setattr(self._tracker, tracker_key, 'default')
         return True

From 43e3c2180d9102c3ea71ddb8dca5699e88d69ebf Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 18 Nov 2024 18:27:24 +0800
Subject: [PATCH 097/200] chore(monitor): use prometheus replace timeseries

---
 charts/controller/templates/_helpers.tpl      |  20 --
 .../templates/controller-secret-creds.yaml    |   3 -
 charts/controller/values.yaml                 |   2 +-
 .../commands/measure_loadbalancers.py         |  16 +-
 .../management/commands/measure_networks.py   |  34 ++-
 rootfs/api/models/config.py                   |   2 +-
 rootfs/api/monitor.py                         | 274 +++++++-----------
 rootfs/api/settings/production.py             |   5 -
 rootfs/api/views.py                           |  59 ++--
 9 files changed, 155 insertions(+), 260 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index d8db87817..968487fd7 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -90,26 +90,6 @@ env:
 - name: DRYCC_DATABASE_REPLICA_URL
   value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/controller"
 {{- end }}
-{{- if (.Values.databaseMonitorUrl) }}
-- name: DRYCC_DATABASE_MONITOR_URL
-  valueFrom:
-    secretKeyRef:
-      name: controller-creds
-      key: database-monitor-url
-{{- else if eq .Values.global.timeseriesLocation "on-cluster"  }}
-- name: DRYCC_TS_USER
-  valueFrom:
-    secretKeyRef:
-      name: timeseries-creds
-      key: user
-- name: DRYCC_TS_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: timeseries-creds
-      key: password
-- name: DRYCC_DATABASE_MONITOR_URL
-  value: "postgres://$(DRYCC_TS_USER):$(DRYCC_TS_PASSWORD)@drycc-timeseries-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/monitor"
-{{- end }}
 {{- if (.Values.workflowManagerUrl) }}
 - name: WORKFLOW_MANAGER_URL
   value: "{{ .Values.workflowManagerUrl }}"
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index fdeb3acfd..bd7f19116 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -11,9 +11,6 @@ data:
   {{- if (.Values.databaseReplicaUrl) }}
   database-replica-url: {{ .Values.databaseReplicaUrl | b64enc }}
   {{- end }}
-  {{- if (.Values.databaseMonitorUrl) }}
-  database-monitor-url: {{ .Values.databaseMonitorUrl | b64enc }}
-  {{- end }}
   {{- if (.Values.rabbitmqUrl) }}
   rabbitmq-url: {{ .Values.rabbitmqUrl | b64enc }}
   {{- end }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 170dbda7e..ae678d72b 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -84,7 +84,7 @@ service:
 # can be specified as key-value pairs under environment
 # this is usually a non required setting.
 environment:
-  RESERVED_NAMES: "drycc, drycc-builder, drycc-monitor-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
+  RESERVED_NAMES: "drycc, drycc-builder, drycc-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
 
 api:
   nodeAffinityPreset:
diff --git a/rootfs/api/management/commands/measure_loadbalancers.py b/rootfs/api/management/commands/measure_loadbalancers.py
index e9c295028..277bdc6a8 100644
--- a/rootfs/api/management/commands/measure_loadbalancers.py
+++ b/rootfs/api/management/commands/measure_loadbalancers.py
@@ -2,6 +2,7 @@
 import time
 import logging
 import ipaddress
+from asgiref.sync import async_to_sync
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -24,9 +25,10 @@ def _measure_loadbalancers(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         loadbalancers = []
-        for item in monitor.query_loadbalancer(app_map.keys(), start, stop):
-            ip = item["ip"]
-            namespace = item["namespace"]
+        for metric, (_, value) in async_to_sync(monitor.query_loadbalancer(
+                app_map.keys(), start, stop)):
+            ip = metric["ip"]
+            namespace = metric["namespace"]
             owner_id = app_map[namespace].owner_id
             loadbalancers.append({
                 "app_id":  str(app_map[namespace].uuid),
@@ -34,12 +36,12 @@ def _measure_loadbalancers(self, app_map, timestamp):
                 "name": self._get_measure_name(ip),
                 "type": "loadbalancer",
                 "unit": "number",
-                "usage": 1,
+                "usage": value,
                 "kwargs": {
                     "ip": ip,
-                    "node": item["node"],
-                    "service": item["service"],
-                    "instance": item["instance"],
+                    "node": metric["node"],
+                    "service": metric["service"],
+                    "instance": metric["instance"],
                 },
                 "timestamp": start
             })
diff --git a/rootfs/api/management/commands/measure_networks.py b/rootfs/api/management/commands/measure_networks.py
index 559452377..013577d77 100644
--- a/rootfs/api/management/commands/measure_networks.py
+++ b/rootfs/api/management/commands/measure_networks.py
@@ -1,6 +1,7 @@
 import uuid
 import time
 import logging
+from asgiref.sync import async_to_sync
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -18,21 +19,34 @@ def _measure_networks(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         networks = []
-        for namespace, pod_name, rx_bytes, tx_bytes in monitor.query_network_flow(
-                app_map.keys(), start, stop):
-            owner_id = app_map[namespace].owner_id
-            base_measure = {
-                "app_id":  str(app_map[namespace].uuid),
-                "owner": owner_id,
+        for metric, (_, value) in async_to_sync(monitor.query_network_receive_flow(
+                app_map.keys(), start, stop)):
+            networks.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "owner": app_map[metric['namespace']].owner_id,
                 "type": "network",
                 "unit": "bytes",
+                "name": "rx",
+                "usage": value,
                 "kwargs": {
-                    "pod": pod_name,
+                    "pod": metric['pod'],
                 },
                 "timestamp": start
-            }
-            networks.append(base_measure | {"name": "rx", "usage": rx_bytes})
-            networks.append(base_measure | {"name": "tx", "usage": tx_bytes})
+            })
+        for metric, (_, value) in async_to_sync(monitor.query_network_transmit_flow(
+                app_map.keys(), start, stop)):
+            networks.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "owner": app_map[metric['namespace']].owner_id,
+                "type": "network",
+                "unit": "bytes",
+                "name": "tx",
+                "usage": value,
+                "kwargs": {
+                    "pod": metric['pod'],
+                },
+                "timestamp": start
+            })
         send_measurements.delay(networks)
 
     def handle(self, *args, **options):
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index afe0b8552..9be81c3b7 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -174,7 +174,7 @@ def _update_values(self, previous_config):
                     else:  # force to string
                         new_item['value'] = str(new_item['value'])
                     break
-            if added:
+            if added and new_item['value'] is not None:
                 data.append(new_item)
         setattr(self, 'values', data)
 
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 244336bf8..e09cc054e 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -1,207 +1,131 @@
 import time
 import aiohttp
-import requests
-from typing import Iterator, Dict
-from contextlib import closing
+from typing import Iterator, AsyncGenerator
 from urllib.parse import urljoin
-from django.db import connections
 from django.conf import settings
 
 
-query_network_flow_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  last(rx_bytes, time) - first(rx_bytes, time) as rx_bytes,
-  last(tx_bytes, time) - first(tx_bytes, time) as tx_bytes
-FROM kubernetes_pod_network
-LEFT OUTER JOIN kubernetes_pod_network_tag
-ON kubernetes_pod_network.tag_id = kubernetes_pod_network_tag.tag_id
-WHERE
-  namespace in ({namespace_range})
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP by namespace, pod_name
+query_last_metrics_promql_tpl = """
+last_over_time({__name__=~"%s",namespace="%s"}[%s])
 """
 
-
-query_container_count_sql_tpl = """
-SELECT
-  count(1)
-FROM (
-  SELECT
-    namespace,
-    pod_name,
-    container_name
-  FROM kubernetes_pod_container
-  LEFT OUTER JOIN kubernetes_pod_container_tag
-  ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id
-  WHERE
-    namespace='{namespace}'
-    AND container_name='{container_name}'
-    AND time > to_timestamp({start})
-    AND time < to_timestamp({stop})
-  GROUP BY namespace, pod_name, container_name, kubernetes_pod_container.tag_id
-) AS container
-GROUP BY namespace, container_name
+query_loadbalancer_promql_tpl = """
+kube_service_status_load_balancer_ingress{namespace=~"%s"}
 """
 
-
-query_cpu_usage_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  container_name,
-  round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  max(cpu_usage_nanocores),
-  round(avg(cpu_usage_nanocores))
-FROM kubernetes_pod_container
-LEFT OUTER JOIN kubernetes_pod_container_tag
-ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id
-WHERE
-  namespace='{namespace}'
-  AND container_name='{container_name}'
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP BY namespace, pod_name, container_name, timestamp
+query_network_receive_flow_promql_tpl = """
+increase(container_network_receive_bytes_total{namespace=~"%s"}[%s])
 """
 
+query_network_transmit_flow_promql_tpl = """
+increase(container_network_transmit_bytes_total{namespace=~"%s"}[%s])
+"""
 
-query_memory_usage_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  container_name,
-  round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  max(memory_working_set_bytes) as max,
-  round(avg(memory_working_set_bytes)) as avg
-FROM kubernetes_pod_container
-LEFT OUTER JOIN kubernetes_pod_container_tag
-ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id
-WHERE
-  namespace='{namespace}'
-  AND container_name='{container_name}'
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP BY namespace, pod_name, container_name, timestamp
+query_cpu_usage_promql_tpl = """
+sum (rate (container_cpu_usage_seconds_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
+by (pod)
 """
 
 
-query_network_usage_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  last(rx_bytes, time) - first(rx_bytes, time) as rx_bytes,
-  last(tx_bytes, time) - first(tx_bytes, time) as tx_bytes
-FROM kubernetes_pod_network
-LEFT OUTER JOIN kubernetes_pod_network_tag
-ON kubernetes_pod_network.tag_id = kubernetes_pod_network_tag.tag_id
-WHERE
-  namespace='{namespace}'
-  AND pod_name like '{pod_name_prefix}%'
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP by namespace, pod_name, timestamp
+query_memory_usage_promql_tpl = """
+sum (rate (container_memory_working_set_bytes{pod=~"^%s-.*$",namespace="%s"}[%s]))
+by (pod)
 """
 
 
-query_loadbalancer_promql_tpl = """
-kube_service_status_load_balancer_ingress{namespace=~"%s"}
+query_network_receive_usage_promql_tpl = """
+sum (rate (container_network_receive_bytes_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
+by (pod)
 """
 
-query_last_metrics_promql_tpl = """
-last_over_time({__name__=~"%s",namespace="%s"}[%s])
+query_network_transmit_usage_promql_tpl = """
+sum (rate (container_network_transmit_bytes_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
+by (pod)
 """
 
 
-async def last_metrics(namespace):
+async def query_prom(url, params) -> list[tuple[dict[str, str], int]]:
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, params=params) as response:
+            if response.status != 200:
+                return []
+            response_json = await response.json()
+            if response_json['status'] != 'success':
+                return []
+            return response_json['data']['result']
+
+
+async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
     if not settings.DRYCC_METRICS_CONFIG:
         return
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
     promql = query_last_metrics_promql_tpl % (
       '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
       namespace,
       settings.DRYCC_METRICS_INTERVAL)
+    for item in await query_prom(url, {"query": promql, "start": int(time.time() - 60)}):
+        yield '%s{%s} %s\n' % (
+            item['metric']['__name__'],
+            ','.join([
+                f'{key}="{value}"' for key, value in item['metric'].items()
+                if key in settings.DRYCC_METRICS_CONFIG[item['metric']['__name__']]
+            ]),
+            item['value'][1]
+        )
+
+
+async def query_loadbalancer(namespaces: Iterator[str], start: int, stop: int
+                             ) -> list[tuple[dict[str, str], int]]:
     url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
-    params = {"query": promql, "start": int(time.time() - 60)}
-    async with aiohttp.ClientSession() as session:
-        async with session.get(url, params=params) as response:
-            if response.status != 200:
-                return
-            items = await response.json()
-            for item in items['data']['result']:
-                yield '%s{%s} %s\n' % (
-                    item['metric']['__name__'],
-                    ','.join([
-                        f'{key}="{value}"' for key, value in item['metric'].items()
-                        if key in settings.DRYCC_METRICS_CONFIG[item['metric']['__name__']]
-                    ]),
-                    item['value'][1]
-                )
-
-
-def query_loadbalancer(namespaces: Iterator[str],
-                       start: int, stop: int) -> Iterator[Dict[str, str]]:
     promql = query_loadbalancer_promql_tpl % "|".join(namespaces)
-    params = {"query": promql, "start": start, "end": stop}
-    response = requests.get(
-        urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query"), params=params)
-    if response.status_code != 200:
-        return
-    yield from (metric["metric"] for metric in response.json()["data"]["result"])
-
-
-def query_network_flow(namespaces: Iterator[str],
-                       start: int, stop: int) -> Iterator[tuple[str, str, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        namespace_range = ', '.join([f"'{namespace}'" for namespace in namespaces])
-        sql = query_network_flow_sql_tpl.format(
-            namespace_range=namespace_range, start=start, stop=stop)
-        cursor.execute(sql)
-        yield from cursor
-
-
-def query_container_count(namespace: str, ptype: str, start: int, stop: int) -> int:
-    with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, ptype)
-        sql = query_container_count_sql_tpl.format(
-            namespace=namespace, container_name=container_name, start=start, stop=stop)
-        cursor.execute(sql)
-        row = cursor.fetchone()
-        return row[0] if row else 0
-
-
-def query_cpu_usage(namespace: str, ptype: str,
-                    start: int, stop: int, every: str
-                    ) -> Iterator[tuple[str, str, str, int, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, ptype)
-        sql = query_cpu_usage_sql_tpl.format(
-            namespace=namespace, container_name=container_name,
-            start=start, stop=stop, every=every)
-        cursor.execute(sql)
-        yield from cursor
-
-
-def query_memory_usage(namespace: str, ptype: str,
-                       start: int, stop: int, every: str
-                       ) -> Iterator[tuple[str, str, str, int, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, ptype)
-        sql = query_memory_usage_sql_tpl.format(
-            namespace=namespace, container_name=container_name,
-            start=start, stop=stop, every=every)
-        cursor.execute(sql)
-        yield from cursor
-
-
-def query_network_usage(namespace: str, ptype: str,
-                        start: int, stop: int, every: str
-                        ) -> Iterator[tuple[str, str, int, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        pod_name_prefix = "%s-%s" % (namespace, ptype)
-        sql = query_network_usage_sql_tpl.format(
-            namespace=namespace, pod_name_prefix=pod_name_prefix,
-            start=start, stop=stop, every=every)
-        cursor.execute(sql)
-        yield from cursor
+    return await query_prom(url, {"query": promql, "start": start, "end": stop})
+
+
+async def query_network_receive_flow(namespaces: Iterator[str], start: int, stop: int
+                                     ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
+    promql = query_network_receive_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
+    return await query_prom(url, {"query": promql, "start": start, "end": stop})
+
+
+async def query_network_transmit_flow(namespaces: Iterator[str], start: int, stop: int
+                                      ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
+    promql = query_network_transmit_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
+    return await query_prom(url, {"query": promql, "start": start, "end": stop})
+
+
+async def query_cpu_usage(namespace: str, ptype: str, every: str,
+                          start: int, stop: int, step: int,
+                          ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    pod_prefix = "%s-%s" % (namespace, ptype)
+    promql = query_cpu_usage_promql_tpl % (pod_prefix, namespace, every)
+    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
+
+
+async def query_memory_usage(namespace: str, ptype: str, every: str,
+                             start: int, stop: int, step: int,
+                             ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    pod_prefix = "%s-%s" % (namespace, ptype)
+    promql = query_memory_usage_promql_tpl % (pod_prefix, namespace, every)
+    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
+
+
+async def query_network_receive_usage(namespace: str, ptype: str, every: str,
+                                      start: int, stop: int, step: int,
+                                      ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    pod_prefix = "%s-%s" % (namespace, ptype)
+    promql = query_network_receive_usage_promql_tpl % (pod_prefix, namespace, every)
+    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
+
+
+async def query_network_transmit_usage(namespace: str, ptype: str, every: str,
+                                       start: int, stop: int, step: int,
+                                       ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    pod_prefix = "%s-%s" % (namespace, ptype)
+    promql = query_network_transmit_usage_promql_tpl % (pod_prefix, namespace, every)
+    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 50ab9699f..1e9311efa 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -372,11 +372,6 @@
 if DRYCC_DATABASE_REPLICA_URL is not None:
     DATABASES["replica"] = dj_database_url.config(default=DRYCC_DATABASE_REPLICA_URL)
 
-# database monitor setting
-DRYCC_DATABASE_MONITOR_URL = os.environ.get('DRYCC_DATABASE_MONITOR_URL', None)
-if DRYCC_DATABASE_MONITOR_URL is not None:
-    DATABASES["monitor"] = dj_database_url.config(default=DRYCC_DATABASE_MONITOR_URL)
-
 # database routers
 DATABASE_ROUTERS = ['api.routers.DefaultReplicaRouter', ]
 
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index c9ada3774..7c218ec78 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -6,6 +6,7 @@
 import logging
 import json
 import requests
+from asgiref.sync import async_to_sync
 from django.db.models import Q
 from django.core.cache import cache
 from django.http import Http404, HttpResponse
@@ -1185,34 +1186,13 @@ def _get_app(self):
         self.check_object_permissions(self.request, app)
         return app
 
-    def _get_cpus(self, app_id, ptype, start, stop, every):
-        avg_list, max_list = [], []
-        for _, _, _, timestamp, max, avg in monitor.query_cpu_usage(
-                app_id, ptype, start, stop, every):
-            max_list.append((timestamp, max))
-            avg_list.append((timestamp, avg))
-        return {
-            "max": max_list,
-            "avg": avg_list,
-        }
-
-    def _get_memory(self, app_id, ptype, start, stop, every):
-        max_list, avg_list = [], []
-        for _, _, _, timestamp, max, avg in monitor.query_memory_usage(
-                app_id, ptype, start, stop, every):
-            max_list.append((timestamp, max))
-            avg_list.append((timestamp, avg))
-        return {
-            "max": max_list,
-            "avg": avg_list,
-        }
-
-    def _get_networks(self, app_id, ptype, start, stop, every):
-        networks = []
-        for _, _, timestamp, rx_bytes, tx_bytes in monitor.query_network_usage(
-                app_id, ptype, start, stop, every):
-            networks.append((timestamp, rx_bytes, tx_bytes))
-        return networks
+    def _get_usage(self, func):
+        def wrap(app_id, ptype, every, start, stop, step):
+            result = {}
+            for item in async_to_sync(func)(app_id, ptype, every, start, stop, step):
+                result[item['metric']['pod']] = item['values']
+            return result
+        return wrap
 
     @method_decorator(cache_page(settings.DRYCC_METRICS_EXPIRY))
     @method_decorator(vary_on_headers("Authorization"))
@@ -1221,19 +1201,22 @@ def status(self, request, **kwargs):
         data = serializers.MetricSerializer(data=self.request.query_params)
         if not data.is_valid():
             return Response(data.errors, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
-        start, stop, every = data.validated_data['start'], data.validated_data[
-            'stop'], data.validated_data["every"]
+        every, start, stop, step = data.validated_data["every"], data.validated_data[
+            'start'], data.validated_data['stop'], data.validated_data['step']
+        params = {
+            "app_id": app_id, "ptype": kwargs['ptype'], "every": every,
+            "start": start, "stop": stop, "step": step,
+        }
         return Response({
             "id": app_id,
             "ptype": kwargs['ptype'],
-            "count": monitor.query_container_count(app_id, kwargs['ptype'], start, stop),
-            "status": {
-                "cpus": self._get_cpus(
-                    app_id, kwargs['ptype'], start, stop, every),
-                "memory": self._get_memory(
-                    app_id, kwargs['ptype'], start, stop, every),
-                "networks": self._get_networks(
-                    app_id, kwargs['ptype'], start, stop, every),
+            "usage": {
+                "cpus": self._get_usage(monitor.query_cpu_usage)(**params),
+                "memory": self._get_usage(monitor.query_memory_usage)(**params),
+                "networks": {
+                    "receive": self._get_usage(monitor.query_network_receive_usage)(**params),
+                    "transmit": self._get_usage(monitor.query_network_transmit_usage)(**params),
+                }
             }
         })
 

From 89ad67c071715ca77d00b6fd3190ed7d414e291c Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 18 Nov 2024 20:53:24 +0800
Subject: [PATCH 098/200] chore(tests): add empty test case

---
 rootfs/api/serializers/__init__.py |  2 ++
 rootfs/api/tests/test_config.py    | 13 +++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index b47920b8c..ad9a0fe23 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -699,6 +699,8 @@ class MetricSerializer(serializers.Serializer):
     stop = serializers.IntegerField(
         min_value=946656000, max_value=4102416000,
         required=False, default=lambda: int(time.time()))
+    step = serializers.IntegerField(
+        min_value=1, max_value=30, required=False, default=14)
     every = serializers.CharField(max_length=50, required=False, default='5m')
 
     def validate(self, attrs):
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index a02613917..eda4ac74d 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -119,6 +119,19 @@ def test_config(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual([], response.data['values'])
+        # repeat settings
+        value2['value'] = "VALUE2"
+        value3['value'] = None
+        body = {'values': [value2, value3]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([value2], response.data['values'])
+        # set empty env
+        value1['value'] = ''
+        body = {'values': [value1]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([value2, value1], response.data['values'])
 
         # disallow put/patch
         response = self.client.put(url)

From 679c08f0d71850372b467b8f443135d1ae94f1b4 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 19 Nov 2024 14:21:45 +0800
Subject: [PATCH 099/200] chore(controller): use valkey replace redis

---
 charts/controller/templates/_helpers.tpl      | 53 +++++++------------
 .../templates/controller-api-deployment.yaml  |  4 +-
 .../controller-celery-deloyment.yaml          |  4 +-
 .../templates/controller-cronjob-daily.yaml   |  4 +-
 .../templates/controller-cronjob-hourly.yaml  |  4 +-
 .../templates/controller-job-upgrade.yaml     |  4 +-
 .../controller-mutate-deloyment.yaml          |  4 +-
 .../templates/controller-secret-creds.yaml    |  7 ++-
 charts/controller/values.yaml                 | 15 +++---
 rootfs/Dockerfile.test                        |  6 +--
 rootfs/api/settings/celery.py                 | 31 +++++++----
 rootfs/api/settings/production.py             | 14 ++---
 ...ice_binding_binding_request_in_flight.json | 14 ++---
 .../admissions/service_binding_ready.json     | 14 ++---
 .../service_instance_provisioning.json        | 14 ++---
 .../admissions/service_instance_ready.json    | 14 ++---
 rootfs/api/tests/test_admissions.py           |  4 +-
 rootfs/bin/test-unit                          | 13 ++---
 rootfs/requirements.txt                       |  2 +-
 19 files changed, 101 insertions(+), 124 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 968487fd7..42b9979a2 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -61,6 +61,21 @@ env:
     secretKeyRef:
       name: builder-key-auth
       key: builder-key
+{{- if (.Values.valkeyUrl) }}
+- name: DRYCC_VALKEY_URL
+  valueFrom:
+    secretKeyRef:
+      name: controller-creds
+      key: valkey-url
+{{- else if eq .Values.global.valkeyLocation "on-cluster"  }}
+- name: VALKEY_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: valkey-creds
+      key: password
+- name: DRYCC_VALKEY_URL
+  value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:16379/0"
+{{- end }}
 {{- if (.Values.databaseReplicaUrl) }}
 - name: DRYCC_DATABASE_REPLICA_URL
   valueFrom:
@@ -102,23 +117,13 @@ env:
   valueFrom:
     fieldRef:
       fieldPath: metadata.namespace
-- name: DRYCC_REDIS_ADDRS
-  valueFrom:
-    secretKeyRef:
-      name: redis-creds
-      key: addrs
-- name: DRYCC_REDIS_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: redis-creds
-      key: password
-{{- if eq .Values.global.rabbitmqLocation "off-cluster" }}
+{{- if (.Values.prometheusUrl) }}
 - name: "DRYCC_PROMETHEUS_URL"
   valueFrom:
     secretKeyRef:
-      name: prometheus-creds
-      key: url
-{{- else }}
+      name: controller-creds
+      key: prometheus-url
+{{- else if eq .Values.global.prometheusLocation "on-cluster" }}
 - name: "DRYCC_PROMETHEUS_USERNAME"
   valueFrom:
     secretKeyRef:
@@ -132,26 +137,6 @@ env:
 - name: "DRYCC_PROMETHEUS_URL"
   value: "http://$(DRYCC_PROMETHEUS_USERNAME):$(DRYCC_PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090"
 {{- end }}
-{{- if (.Values.rabbitmqUrl) }}
-- name: DRYCC_RABBITMQ_URL
-  valueFrom:
-    secretKeyRef:
-      name: controller-creds
-      key: rabbitmq-url
-{{- else if eq .Values.global.rabbitmqLocation "on-cluster" }}
-- name: "DRYCC_RABBITMQ_USERNAME"
-  valueFrom:
-    secretKeyRef:
-      name: rabbitmq-creds
-      key: username
-- name: "DRYCC_RABBITMQ_PASSWORD"
-  valueFrom:
-    secretKeyRef:
-      name: rabbitmq-creds
-      key: password
-- name: "DRYCC_RABBITMQ_URL"
-  value: "amqp://$(DRYCC_RABBITMQ_USERNAME):$(DRYCC_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/controller"
-{{- end }}
 {{- if eq .Values.global.passportLocation "on-cluster"}}
 - name: "DRYCC_PASSPORT_URL"
 {{- if .Values.global.certManagerEnabled }}
diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/controller-api-deployment.yaml
index 7f61b98cb..d5b1971f4 100644
--- a/charts/controller/templates/controller-api-deployment.yaml
+++ b/charts/controller/templates/controller-api-deployment.yaml
@@ -34,9 +34,7 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-        - -a
-        - $(DRYCC_REDIS_ADDRS)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/controller-celery-deloyment.yaml
index cbe57dbcc..d2d1f5cc0 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/controller-celery-deloyment.yaml
@@ -34,9 +34,9 @@ spec:
           - netcat
           - -v
           - -u
-          - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+          - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
           - -a
-          - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+          - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller-celery
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index d97446e5e..365149f9d 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -29,9 +29,9 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
               - -a
-              - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+              - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
             {{- include "controller.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 94c73ff50..4bec8c2d4 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -29,9 +29,9 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
               - -a
-              - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+              - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
             {{- include "controller.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index 4c09e3f8d..af78741e2 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -18,9 +18,7 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-        - -a
-        - $(DRYCC_REDIS_ADDRS)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller-job-upgrade
diff --git a/charts/controller/templates/controller-mutate-deloyment.yaml b/charts/controller/templates/controller-mutate-deloyment.yaml
index d30605770..def3c3ce0 100644
--- a/charts/controller/templates/controller-mutate-deloyment.yaml
+++ b/charts/controller/templates/controller-mutate-deloyment.yaml
@@ -24,9 +24,9 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         - -a
-        - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+        - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index bd7f19116..79684193f 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -5,14 +5,17 @@ metadata:
   labels:
     heritage: drycc
 data:
+  {{- if (.Values.valkeyUrl) }}
+  valkey-url: {{ .Values.valkeyUrl | b64enc }}
+  {{- end }}
   {{- if (.Values.databaseUrl) }}
   database-url: {{ .Values.databaseUrl | b64enc }}
   {{- end }}
   {{- if (.Values.databaseReplicaUrl) }}
   database-replica-url: {{ .Values.databaseReplicaUrl | b64enc }}
   {{- end }}
-  {{- if (.Values.rabbitmqUrl) }}
-  rabbitmq-url: {{ .Values.rabbitmqUrl | b64enc }}
+  {{- if (.Values.prometheusUrl) }}
+  prometheus-url: {{ .Values.prometheusUrl | b64enc }}
   {{- end }}
   {{- if (.Values.passportUrl) }}
   passport-url: {{ .Values.passportUrl | b64enc }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index ae678d72b..d0a4e90e8 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -54,6 +54,8 @@ mutateReplicas: 1
 # Forbid: The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
 # Replace: If it is time for a new job run and the previous job run hasn't finished yet, the cron job replaces the currently running job run with a new job run
 concurrencyPolicy: "Replace"
+## valkeyUrl is will no longer use the built-in valkey component
+valkeyUrl: ""
 # databaseUrl and databaseReplicaUrl are will no longer use the built-in database component
 databaseUrl: ""
 databaseReplicaUrl: ""
@@ -66,9 +68,6 @@ workflowManagerUrl: ""
 workflowManagerAccessKey: ""
 workflowManagerSecretKey: ""
 
-# Configuring this will no longer use the built-in rabbitmq component
-rabbitmqUrl: ""
-
 # limit specs, plans config
 config:
   metrics: ""
@@ -131,9 +130,6 @@ mutate:
     extraMatchLabels:
       component: "drycc-controller-mutate"
 
-redis:
-  replicas: 1
-
 storage:
   csi:
     storageClassName: drycc-storage
@@ -157,7 +153,6 @@ global:
   #   configured above)
   # - off-cluster: Run PostgreSQL outside the Kubernetes cluster (configure in database section)
   databaseLocation: "on-cluster"
-
   # Set the location of Workflow's Registry
   #
   # Valid values are:
@@ -186,3 +181,9 @@ global:
   # Whether cert_manager is enabled to automatically generate controller certificates
   certManagerEnabled: true
   passportLocation: "on-cluster"
+  # Set the location of Workflow's prometheus cluster
+  #
+  # Valid values are:
+  # - on-cluster: Run prometheus within the Kubernetes cluster
+  # - off-cluster: prometheus is running outside of the cluster and credentials and connection information will be provided.
+  prometheusLocation: "on-cluster"
\ No newline at end of file
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index 9d43c0b03..099f9444a 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -8,8 +8,7 @@ COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
 
 ENV PGDATA="/opt/drycc/postgresql/data" \
   PYTHON_VERSION="3.12" \
-  REDIS_VERSION="7.2.4" \
-  RABBITMQ_VERSION="3.12.13" \
+  VALKEY_VERSION="8.0.1" \
   POSTGRES_VERSION="15.5" \
   GOSU_VERSION="1.17"
 
@@ -18,8 +17,7 @@ RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
   && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \
   && chmod +x /usr/local/bin/codecov \
   && install-stack python $PYTHON_VERSION \
-  && install-stack redis $REDIS_VERSION \
-  && install-stack rabbitmq $RABBITMQ_VERSION \
+  && install-stack valkey $VALKEY_VERSION \
   && install-stack postgresql $POSTGRES_VERSION \
   && install-stack gosu $GOSU_VERSION && . init-stack \
   && python3 -m venv ${DRYCC_HOME_DIR}/.venv \
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
index 78c33ed44..9485d238a 100644
--- a/rootfs/api/settings/celery.py
+++ b/rootfs/api/settings/celery.py
@@ -5,7 +5,6 @@
 
 class Config(object):
     # Celery Configuration Options
-    timezone = "Asia/Shanghai"
     enable_utc = True
     task_serializer = 'pickle'
     accept_content = frozenset([
@@ -17,12 +16,15 @@ class Config(object):
     task_track_started = True
     task_time_limit = 30 * 60
     worker_max_tasks_per_child = 200
+    worker_prefetch_multiplier = 1
     result_expires = 24 * 60 * 60
-    broker_url = os.environ.get('DRYCC_RABBITMQ_URL', 'amqp://guest:guest@127.0.0.1:5672/')  # noqa
     cache_backend = 'django-cache'
-    task_default_queue = 'controller.low'
+    task_default_queue = 'controller.middle'
     task_default_exchange = 'controller.priority'
-    task_default_routing_key = 'controller.priority.low'
+    task_default_routing_key = 'controller.priority.middle'
+    broker_transport_options = {"queue_order_strategy": "sorted"}
+    task_create_missing_queues = True
+    task_inherit_parent_priority = True
     broker_connection_retry_on_startup = True
     worker_cancel_long_running_tasks_on_connection_loss = True
 
@@ -30,7 +32,12 @@ class Config(object):
 app = Celery('drycc')
 app.config_from_object(Config())
 app.conf.update(
+    timezone=os.environ.get('TZ', 'UTC'),
     task_routes={
+        'api.tasks.send_measurements': {
+            'queue': 'controller.low',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.low',
+        },
         'api.tasks.scale_app': {
             'queue': 'controller.high',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
@@ -51,24 +58,26 @@ class Config(object):
             'queue': 'controller.high',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
-        'api.tasks.send_measurements': {
-            'queue': 'controller.middle',
-            'exchange': 'controller.priority', 'routing_key': 'controller.priority.middle',
-        },
     },
     task_queues=(
         Queue(
             'controller.low', exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.low', queue_arguments={'x-queue-type': 'quorum'},
+            routing_key='controller.priority.low',
         ),
         Queue(
             'controller.high', exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.high', queue_arguments={'x-queue-type': 'quorum'},
+            routing_key='controller.priority.high',
         ),
         Queue(
             'controller.middle', exchange=Exchange('controller.priority', type="direct"),
-            routing_key='controller.priority.middle', queue_arguments={'x-queue-type': 'quorum'},
+            routing_key='controller.priority.middle',
         ),
     ),
 )
+DRYCC_VALKEY_URL = os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379')
+app.conf.update(
+    broker_url=DRYCC_VALKEY_URL,
+    result_backend=DRYCC_VALKEY_URL,
+    broker_transport_options={"queue_order_strategy": "sorted", "visibility_timeout": 43200},
+)
 app.autodiscover_tasks()
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 1e9311efa..876c90c30 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -8,7 +8,6 @@
 import tempfile
 import dj_database_url
 
-
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.join(BASE_DIR, 'apps_extra'))
 
@@ -438,20 +437,13 @@
 AUTHENTICATION_BACKENDS = ("api.backend.DryccOIDC", ) + AUTHENTICATION_BACKENDS
 DRYCC_CACHE_USER_TIME = int(os.environ.get('DRYCC_CACHE_USER_TIME', 30 * 60))
 
-# Redis Configuration
-DRYCC_REDIS_ADDRS = os.environ.get('DRYCC_REDIS_ADDRS', '127.0.0.1:6379').split(",")
-DRYCC_REDIS_PASSWORD = os.environ.get('DRYCC_REDIS_PASSWORD', '')
-
-# Cache Configuration
+# Cache Valkey Configuration
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": [
-            'redis://:{}@{}'.format(
-                DRYCC_REDIS_PASSWORD, DRYCC_REDIS_ADDR) for DRYCC_REDIS_ADDR in DRYCC_REDIS_ADDRS
-        ],
+        "LOCATION": os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379'),
         "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.ShardClient",
+            "CLIENT_CLASS": "django_redis.client.DefaultClient",
         }
     }
 }
diff --git a/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json b/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json
index 85b4024a4..5dc9641cc 100644
--- a/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json
+++ b/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json
@@ -25,7 +25,7 @@
 			"resource": "servicebindings"
 		},
 		"requestSubResource": "status",
-		"name": "redis-t2",
+		"name": "valkey-t2",
 		"namespace": "myapp",
 		"operation": "UPDATE",
 		"userInfo": {
@@ -116,7 +116,7 @@
 						"time": "2024-03-25T01:48:21Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74043277",
 				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
@@ -124,9 +124,9 @@
 			"spec": {
 				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
 				"instanceRef": {
-					"name": "redis-t2"
+					"name": "valkey-t2"
 				},
-				"secretName": "redis-t2",
+				"secretName": "valkey-t2",
 				"userInfo": {
 					"extra": {
 						"authentication.kubernetes.io/pod-name": [
@@ -238,7 +238,7 @@
 						"time": "2024-03-25T01:48:21Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74043277",
 				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
@@ -246,9 +246,9 @@
 			"spec": {
 				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
 				"instanceRef": {
-					"name": "redis-t2"
+					"name": "valkey-t2"
 				},
-				"secretName": "redis-t2",
+				"secretName": "valkey-t2",
 				"userInfo": {
 					"extra": {
 						"authentication.kubernetes.io/pod-name": [
diff --git a/rootfs/api/tests/admissions/service_binding_ready.json b/rootfs/api/tests/admissions/service_binding_ready.json
index 9230ea0ff..d486b302b 100644
--- a/rootfs/api/tests/admissions/service_binding_ready.json
+++ b/rootfs/api/tests/admissions/service_binding_ready.json
@@ -25,7 +25,7 @@
 			"resource": "servicebindings"
 		},
 		"requestSubResource": "status",
-		"name": "redis-t2",
+		"name": "valkey-t2",
 		"namespace": "myapp",
 		"operation": "UPDATE",
 		"userInfo": {
@@ -114,7 +114,7 @@
 						"time": "2024-03-25T01:48:22Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74043278",
 				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
@@ -122,9 +122,9 @@
 			"spec": {
 				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
 				"instanceRef": {
-					"name": "redis-t2"
+					"name": "valkey-t2"
 				},
-				"secretName": "redis-t2",
+				"secretName": "valkey-t2",
 				"userInfo": {
 					"extra": {
 						"authentication.kubernetes.io/pod-name": [
@@ -250,7 +250,7 @@
 						"time": "2024-03-25T01:48:21Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74043278",
 				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
@@ -258,9 +258,9 @@
 			"spec": {
 				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
 				"instanceRef": {
-					"name": "redis-t2"
+					"name": "valkey-t2"
 				},
-				"secretName": "redis-t2",
+				"secretName": "valkey-t2",
 				"userInfo": {
 					"extra": {
 						"authentication.kubernetes.io/pod-name": [
diff --git a/rootfs/api/tests/admissions/service_instance_provisioning.json b/rootfs/api/tests/admissions/service_instance_provisioning.json
index 444359a58..fd4f68f17 100644
--- a/rootfs/api/tests/admissions/service_instance_provisioning.json
+++ b/rootfs/api/tests/admissions/service_instance_provisioning.json
@@ -25,7 +25,7 @@
 			"resource": "serviceinstances"
 		},
 		"requestSubResource": "status",
-		"name": "redis-t2",
+		"name": "valkey-t2",
 		"namespace": "myapp",
 		"operation": "UPDATE",
 		"userInfo": {
@@ -141,13 +141,13 @@
 						"time": "2024-03-25T01:42:31Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74040375",
 				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
 			},
 			"spec": {
-				"clusterServiceClassExternalName": "redis",
+				"clusterServiceClassExternalName": "valkey",
 				"clusterServiceClassRef": {
 					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
 				},
@@ -215,7 +215,7 @@
 				"orphanMitigationInProgress": false,
 				"provisionStatus": "",
 				"reconciledGeneration": 0,
-				"userSpecifiedClassName": "ClusterServiceClass/redis",
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
 				"userSpecifiedPlanName": "standard-128"
 			}
 		},
@@ -315,13 +315,13 @@
 						"time": "2024-03-25T01:42:31Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74040375",
 				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
 			},
 			"spec": {
-				"clusterServiceClassExternalName": "redis",
+				"clusterServiceClassExternalName": "valkey",
 				"clusterServiceClassRef": {
 					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
 				},
@@ -389,7 +389,7 @@
 				"orphanMitigationInProgress": false,
 				"provisionStatus": "",
 				"reconciledGeneration": 0,
-				"userSpecifiedClassName": "ClusterServiceClass/redis",
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
 				"userSpecifiedPlanName": "standard-128"
 			}
 		},
diff --git a/rootfs/api/tests/admissions/service_instance_ready.json b/rootfs/api/tests/admissions/service_instance_ready.json
index 07a2b86e3..60f833f3a 100644
--- a/rootfs/api/tests/admissions/service_instance_ready.json
+++ b/rootfs/api/tests/admissions/service_instance_ready.json
@@ -25,7 +25,7 @@
 			"resource": "serviceinstances"
 		},
 		"requestSubResource": "status",
-		"name": "redis-t2",
+		"name": "valkey-t2",
 		"namespace": "myapp",
 		"operation": "UPDATE",
 		"userInfo": {
@@ -139,13 +139,13 @@
 						"time": "2024-03-25T01:44:38Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74040375",
 				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
 			},
 			"spec": {
-				"clusterServiceClassExternalName": "redis",
+				"clusterServiceClassExternalName": "valkey",
 				"clusterServiceClassRef": {
 					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
 				},
@@ -211,7 +211,7 @@
 				"orphanMitigationInProgress": false,
 				"provisionStatus": "Provisioned",
 				"reconciledGeneration": 2,
-				"userSpecifiedClassName": "ClusterServiceClass/redis",
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
 				"userSpecifiedPlanName": "standard-128"
 			}
 		},
@@ -311,13 +311,13 @@
 						"time": "2024-03-25T01:42:31Z"
 					}
 				],
-				"name": "redis-t2",
+				"name": "valkey-t2",
 				"namespace": "myapp",
 				"resourceVersion": "74040375",
 				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
 			},
 			"spec": {
-				"clusterServiceClassExternalName": "redis",
+				"clusterServiceClassExternalName": "valkey",
 				"clusterServiceClassRef": {
 					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
 				},
@@ -385,7 +385,7 @@
 				"orphanMitigationInProgress": false,
 				"provisionStatus": "",
 				"reconciledGeneration": 0,
-				"userSpecifiedClassName": "ClusterServiceClass/redis",
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
 				"userSpecifiedPlanName": "standard-128"
 			}
 		},
diff --git a/rootfs/api/tests/test_admissions.py b/rootfs/api/tests/test_admissions.py
index 20b2d701c..00b9463ec 100644
--- a/rootfs/api/tests/test_admissions.py
+++ b/rootfs/api/tests/test_admissions.py
@@ -64,10 +64,10 @@ def test_admissions_scale(self, requests_mock):
 
     def test_admissions_service_catalog(self, requests_mock):
         app_id = self.create_app("myapp")
-        resource_name = 'redis-t2'
+        resource_name = 'valkey-t2'
         response = self.client.post(
             '/v2/apps/{}/resources'.format(app_id),
-            data={'name': resource_name, 'plan': 'redis:standard-128'}
+            data={'name': resource_name, 'plan': 'valkey:standard-128'}
         )
         self.assertEqual(response.status_code, 201, response.data)
 
diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit
index 54d2ba37c..45c343d88 100755
--- a/rootfs/bin/test-unit
+++ b/rootfs/bin/test-unit
@@ -6,20 +6,13 @@
 # fail hard and fast even on pipelines
 set -eou pipefail
 
-function start_redis() {
+function start_valkey() {
     cd /tmp
-    nohup redis-server > /var/log/redis.log 2>&1 &
-    cd -
-}
-
-function start_rabbitmq() {
-    cd /tmp
-    nohup rabbitmq-server > /var/log/rabbitmq.log 2>&1 &
+    nohup valkey-server > /var/log/valkey.log 2>&1 &
     cd -
 }
 
 gosu postgres pg_ctl -D "$PGDATA" start && ln -s /tmp/.s.PGSQL.5432 /var/run/postgresql/.s.PGSQL.5432
-start_redis
-start_rabbitmq
+start_valkey
 python3 manage.py check
 coverage run --source=api,scheduler manage.py test --settings=api.settings.testing --noinput api scheduler.tests --exclude-tag=auth
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 30a762c64..65c4082a2 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -22,7 +22,7 @@ ndg-httpsclient==0.5.1
 pytz==2024.2
 requests==2.32.3
 requests-toolbelt==1.0.0
-celery==5.5.0rc1
+celery==5.4.0
 hiredis==3.0.0
 django_redis==5.4.0
 dj-database-url==2.3.0

From 805c62c8cadfd35f42bed8b7aa5f9bc592a8d0c1 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 27 Nov 2024 16:30:03 +0800
Subject: [PATCH 100/200] chore(filer): add clean old filer

---
 rootfs/api/filer.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
index daff5a475..0e7152ebc 100644
--- a/rootfs/api/filer.py
+++ b/rootfs/api/filer.py
@@ -1,4 +1,5 @@
 import uuid
+import logging
 import requests
 from django.conf import settings
 from django.core.cache import cache
@@ -6,6 +7,8 @@
 
 from .utils import random_string, get_session, CacheLock
 
+logger = logging.getLogger(__name__)
+
 
 class FilerClient(object):
 
@@ -16,6 +19,9 @@ def __init__(self, app_id, volume, scheduler):
         self.volume = volume
         self.scheduler = scheduler
 
+    def log(self, message, level=logging.INFO):
+        logger.log(level, "[{}]: {}".format(self.app_id, message))
+
     @property
     def server(self):
         lock_key = f"filer:lock:{self.app_id}:{self.volume.name}"
@@ -35,6 +41,7 @@ def cache_key(self):
         return f"filer:{self.app_id}:{self.volume.name}"
 
     def get_server(self):
+        self.clean()  # clean old filer
         pod_name = f"drycc-filer-{uuid.uuid4().hex}"
         k8s_volume = {"name": self.volume.name}
         if self.volume.type == "csi":
@@ -59,6 +66,19 @@ def get_server(self):
         address = self.scheduler.pod.get(self.app_id, pod_name).json()["status"]["podIP"]
         return {"address": address, "username": username, "password": password}
 
+    def clean(self):
+        response = self.scheduler.pod.get(
+            self.app_id, labels={"app": self.app_id, "type": "filer"})
+        if response.status_code != 200:
+            self.log("clean up old filter errors")
+            return False
+        for item in response.json()["items"]:
+            if item['status']['phase'] in ('Succeeded', 'Failed'):
+                pod_name = item['metadata']['name']
+                self.scheduler.pod.delete(self.app_id, pod_name)
+        self.log("clean up old filter completed")
+        return True
+
     def health(self, server):
         try:
             return self.request(

From 6ae8bfb210ed8446695ce4cb5b39be875bf7e280 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 28 Nov 2024 13:26:32 +0800
Subject: [PATCH 101/200] chore(ps): support previous pod logs (#171)

---
 rootfs/api/consumers.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index e1552043d..da36f4b05 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -143,10 +143,12 @@ async def receive(self, text_data=None, bytes_data=None):
         args = (self.pod_id, self.id)
         lines = data.get("lines", 300)
         follow = data.get("follow", False)
+        previous = data.get("previous", False)
         kwargs = {
             "follow": follow,
             "container": data.get("container", ""),
             "_preload_content": not follow,
+            "previous": previous
         }
         if lines > 0:
             kwargs["tail_lines"] = lines

From ce6d45a12f617d03b260decc66015ec4443c1be3 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 2 Dec 2024 15:02:11 +0800
Subject: [PATCH 102/200] chore(gateway): add parent ref filter

---
 rootfs/api/models/gateway.py     |  6 ++-
 rootfs/api/tests/test_gateway.py | 80 ++++++++++++++++++++++++++++++++
 2 files changed, 84 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index ca42d1091..ad0aad94e 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -404,14 +404,16 @@ def _get_all_parent_refs(self):
         for gateway in self.app.gateway_set.filter(
                 name__in=[item["name"] for item in self.parent_refs]):
             gateways[gateway.name] = gateway
-        parent_refs, http_parent_refs = [], []
+        hostnames, parent_refs, http_parent_refs = self.hostnames, [], []
         for item in self.parent_refs:
             gateway_name, port = item["name"], item["port"]
             if gateway_name not in gateways:
                 continue
             gateway = gateways[gateway_name]
             for listener in gateway.listeners:
-                if listener["port"] == port and listener["protocol"] in self.protocols:
+                hostname = listener.get('hostname', None)
+                if listener["port"] == port and listener["protocol"] in self.protocols and (
+                        hostname is None or hostname in hostnames):
                     parent_ref = {
                         "group": "gateway.networking.k8s.io",
                         "kind": "Gateway",
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 32e1569de..7bbc13cb0 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -703,3 +703,83 @@ def test_change_tls(self):
         }]
         self.assertEqual(len(response.data["results"]), 1, response.data)
         self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+
+    def test_route_parent_refs(self):
+        app_id = self.create_app_with_domain_and_deploy()
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }]
+        # enable tls
+        self.change_certs_auto(app_id, True)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }, {
+            'name': f'{app_id}',
+            'port': 443
+        }]
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+        # add new domain
+        ptype, kind, route_name, domain = (
+            'gateway', "HTTPRoute", "myroute-1", 'test-domain-gateway.example.com')
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': domain, 'ptype': ptype}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        # add new services
+        response = self.client.post(
+            '/v2/apps/{}/services'.format(app_id),
+            {
+                'port': 9999,
+                'protocol': 'TCP',
+                'target_port': 9999,
+                'ptype': 'gateway'
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        # add new router
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "kind": kind,
+                "name": route_name,
+                "rules": [{
+                    "backendRefs": [{
+                        "kind": "Service",
+                        "name": f"{app_id}-{ptype}",
+                        "port": 9999,
+                        "weight": 100,
+                    }],
+                }],
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.client.patch(
+            '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
+            {
+                "gateway": app_id,
+                "port": 443
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        route = Route.objects.get(name=route_name)
+        response = route.scheduler().httproutes.get(app_id, route_name)
+        self.assertEqual(response.status_code, 200, response.json())
+        self.assertEqual(len(response.json()['spec']['parentRefs']), 1)
+        # get gateway
+        response = route.scheduler().gateways.get(app_id, app_id)
+        listeners = [
+            listener for listener in response.json()['spec']['listeners']
+            if listener['port'] == 443
+        ]
+        self.assertEqual(len(listeners) > 1,  True)
+        hostnames = [listener['hostname'] for listener in listeners]
+        self.assertEqual(domain in hostnames,  True)

From 6ec1823d8ad784f13dcf58e7e57ea0c3a2a33064 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Tue, 10 Dec 2024 09:16:57 +0800
Subject: [PATCH 103/200] chore(controller): modify query latest release (#173)

* chore(controller): modify query latest release

* chore(gateway): listener filter

* chore(config): do not set empty string
---
 rootfs/api/models/app.py         | 14 +++++-----
 rootfs/api/models/build.py       |  5 ++--
 rootfs/api/models/config.py      |  9 ++++---
 rootfs/api/models/gateway.py     |  8 +-----
 rootfs/api/models/release.py     | 31 ++++++++++++----------
 rootfs/api/tests/test_gateway.py | 44 +++++++++++++++++++++++---------
 rootfs/api/views.py              | 12 ++++-----
 7 files changed, 71 insertions(+), 52 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 426637cbb..429304f6c 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -269,7 +269,7 @@ def restart(self, **kwargs):  # noqa
 
     def scale(self, user, structure):
         err_msg = None
-        release = self.release_set.filter(failed=False).latest()
+        release = Release.latest(self)
         if (PTYPE_RUN in structure or release.build is None):
             if PTYPE_RUN in structure:
                 err_msg = 'Cannot set scale for reserved types, procfile type is: run'
@@ -348,9 +348,9 @@ def deploy(self, release, ptypes=None, force_deploy=False, rollback_on_failure=T
         release.clean(ptypes)
 
     def mount(self, user, volume, structure=None):
-        if self.release_set.filter(failed=False).latest().build is None:
+        release = Release.latest(self)
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release')
-        release = self.release_set.filter(failed=False).latest()
         app_settings = self.appsettings_set.latest()
         self._mount(user, volume, release, app_settings, structure=structure)
 
@@ -387,8 +387,8 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             return ''.join(random.choice(chars) for _ in range(size))
 
         """Run a one-off command in an ephemeral app container."""
-        release = self.release_set.filter(failed=False).latest()
-        if release.build is None:
+        release = Release.latest(self)
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release to run this command')
 
         app_settings = self.appsettings_set.latest()
@@ -656,8 +656,8 @@ def image_pull_secret(self, namespace, registry, image):
         return name
 
     def state_to_k8s(self):
-        release = self.release_set.filter(failed=False).latest()
-        if release.build is None:
+        release = Release.latest(self)
+        if release is None or release.build is None:
             self.log('the last release does not have a build, skipping deployment...')
             return
         ptypes = set()
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index f00b2b58f..663ca4851 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -4,6 +4,7 @@
 from api.exceptions import DryccException
 from .base import UuidAuditedModel, PTYPE_WEB
 from .config import Config
+from .release import Release
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -72,7 +73,7 @@ def get_image(self, ptype, default_image=None):
         return default_image if default_image else self.image
 
     def create_release(self, user, *args, **kwargs):
-        latest_release = self.app.release_set.filter(failed=False).latest()
+        latest_release = Release.latest(self.app)
         try:
             new_release = latest_release.new(
                 user,
@@ -109,7 +110,7 @@ def _get_or_create_config(self):
         """
         dryccfile to config
         """
-        latest_release = self.app.release_set.filter(failed=False).latest()
+        latest_release = Release.latest(self.app)
         config_values, config_values_ref, config_healthcheck = [], {}, {}
         for group, values in self.dryccfile.get('config', {}).items():
             for value in values:
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 9be81c3b7..7b57ae068 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -125,9 +125,10 @@ def save(self, ignore_update_fields=None, *args, **kwargs):
         """merge the old config with the new"""
         try:
             # Get config from the latest available release
-            try:
-                previous_config = self.app.release_set.filter(failed=False).latest().config
-            except Release.DoesNotExist:
+            latest_releases = Release.latest(self.app)
+            if latest_releases:
+                previous_config = latest_releases.config
+            else:
                 # If that doesn't exist then fallback on app config
                 # usually means a totally new app
                 previous_config = self.app.config_set.latest()
@@ -174,7 +175,7 @@ def _update_values(self, previous_config):
                     else:  # force to string
                         new_item['value'] = str(new_item['value'])
                     break
-            if added and new_item['value'] is not None:
+            if added and new_item['value'] is not None and new_item['value'] != "":
                 data.append(new_item)
         setattr(self, 'values', data)
 
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index ad0aad94e..9312f63b3 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -64,7 +64,7 @@ def addresses(self):
     def listeners(self):
         auto_tls = self.app.tls_set.latest().certs_auto_enabled
         listeners = []
-        domains = list(self._get_tls_domain(auto_tls))
+        domains = list(self.app.domain_set.all())
         for item in self.ports:
             port, protocol = item["port"], item["protocol"]
             if item["protocol"] in HOSTNAME_PROTOCOLS:
@@ -146,12 +146,6 @@ def _check_port(self, port, protocol):
                     return False
         return True
 
-    def _get_tls_domain(self, auto_tls):
-        domains = self.app.domain_set.all()
-        if not auto_tls:
-            domains = domains.exclude(certificate=None)
-        return domains
-
     def _get_listener_name(self, port, protocol, index):
         if protocol in ("TCP", "TLS", "HTTP"):
             protocol = "TCP"
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index aeb0898c1..d85fcae97 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -211,28 +211,31 @@ def deploy(self, ptypes=None, force_deploy=False):
             raise DryccException(f"{msg}: {lock.locked(ptypes)}")
         run_pipeline.delay(self, ptypes, force_deploy)
 
-    def previous(self, state="succeed"):
-        """
-        Return the previous Release to this one.
-
-        :return: the previous :class:`Release`, or None
-        """
-        releases = self.app.release_set
-        if self.pk:
-            releases = releases.exclude(pk=self.pk)
+    @classmethod
+    def latest(cls, app, state=None, exclude_pk=None):
+        releases = app.release_set
+        if exclude_pk:
+            releases = releases.exclude(pk=exclude_pk)
         q = Q(failed=False, state="succeed") if state else Q(failed=False)
         try:
-            app_settings = self.app.appsettings_set.latest()
+            app_settings = app.appsettings_set.latest()
             if app_settings.autorollback is False:
-                q = Q(state__in=["crashed", "succeed"])
+                q = Q()
         except AppSettings.DoesNotExist:
             pass
         try:
             # Get the Release previous to this one
-            prev_release = releases.filter(q).latest()
+            release = releases.filter(q).latest()
         except Release.DoesNotExist:
-            prev_release = None
-        return prev_release
+            release = None
+        return release
+
+    def previous(self, state="succeed"):
+        """
+        Return the previous Release to this one.
+        :return: the previous :class:`Release`, or None
+        """
+        return self.latest(self.app, state, self.pk)
 
     def rollback(self, user, ptypes=None, version=None):
         try:
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 7bbc13cb0..f0904fea6 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -328,14 +328,24 @@ def test_change_tls(self):
         app_id = self.create_app_with_domain_and_deploy()
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
         expect = [{
-            'allowedRoutes': {
-                'namespaces': {
-                    'from': 'All'
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
                 }
             },
-            'name': 'tcp-80-0',
-            'port': 80,
-            'protocol': 'HTTP'
+            "name": "tcp-80-1",
+            "port": 80,
+            "hostname": "test-domain.example.com",
+            "protocol": "HTTP"
+        }, {
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
+                }
+            },
+            "name": "tcp-80-0",
+            "port": 80,
+            "protocol": "HTTP"
         }]
         self.assertEqual(response.data["count"], 1, response.data)
         self.assertEqual(response.data["results"][0]["listeners"], expect, response.data)
@@ -377,14 +387,24 @@ def test_change_tls(self):
         self.change_certs_auto(app_id, False)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
         expect = [{
-            'allowedRoutes': {
-                'namespaces': {
-                    'from': 'All'
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
+                }
+            },
+            "name": "tcp-80-1",
+            "port": 80,
+            "hostname": "test-domain.example.com",
+            "protocol": "HTTP"
+        }, {
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
                 }
             },
-            'name': 'tcp-80-0',
-            'port': 80,
-            'protocol': 'HTTP'
+            "name": "tcp-80-0",
+            "port": 80,
+            "protocol": "HTTP"
         }]
         self.assertEqual(response.data["count"], 1, response.data)
         self.assertEqual(response.json()["results"][0]["listeners"], expect, response.data)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 7c218ec78..88720daca 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -279,7 +279,7 @@ def get_object(self):
             release = get_object_or_404(
                 models.release.Release, app=self.get_app(), version=int(version))
         else:
-            release = self.get_app().release_set.filter(failed=False).latest()
+            release = models.release.Release.latest(self.get_app())
         return getattr(release, self.model.__name__.lower())
 
 
@@ -321,7 +321,7 @@ def run(self, request, **kwargs):
             expires = settings.KUBERNETES_JOB_MAX_TTL_SECONDS_AFTER_FINISHED
         if not command:
             raise DryccException('command is a required field, or it can be defined in Procfile')
-        release = app.release_set.filter(failed=False).latest()
+        release = models.release.Release.latest(app)
         if release.build is None:
             raise DryccException('no build available, please deploy a release')
         volumes = request.data.get('volumes', None)
@@ -417,7 +417,7 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_200_OK)
 
     def post_save(self, config):
-        latest_release = config.app.release_set.filter(failed=False).latest()
+        latest_release = models.release.Release.latest(self.get_app())
         try:
             release = latest_release.new(
                 self.request.user, config=config, build=latest_release.build)
@@ -673,7 +673,7 @@ def rollback(self, request, **kwargs):
         Create a new release as a copy of the state of the compiled slug and config vars of a
         previous release.
         """
-        latest_release = self.get_app().release_set.filter(failed=False).latest()
+        latest_release = models.release.Release.latest(self.get_app())
         ptypes = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
         ptypes = latest_release.app.check_ptypes(ptypes)
@@ -786,7 +786,7 @@ def create(self, request, *args, **kwargs):
         # return the application databag
         response = {
             'release': {
-                'version': app.release_set.filter(failed=False).latest().version
+                'version': models.release.Release.latest(app).version
             }
         }
         return Response(response, status=status.HTTP_200_OK)
@@ -807,7 +807,7 @@ def create(self, request, *args, **kwargs):
         has_permission, message = permissions.has_app_permission(request.user, app, request.method)
         if not has_permission:
             return Response(message, status=status.HTTP_403_FORBIDDEN)
-        config = app.release_set.filter(failed=False).latest().config
+        config = models.release.Release.latest(app).config
         serializer = self.get_serializer(config)
         return Response(serializer.data, status=status.HTTP_200_OK)
 

From bfe13e5abb3db0b3376e6715759b19ef540c0352 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 10 Dec 2024 09:17:12 +0800
Subject: [PATCH 104/200] chore(deps): bump django from 4.2.16 to 4.2.17 in
 /rootfs (#172)

Bumps [django](https://github.com/django/django) from 4.2.16 to 4.2.17.
- [Commits](https://github.com/django/django/compare/4.2.16...4.2.17)

---
updated-dependencies:
- dependency-name: django
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 65c4082a2..b017d2244 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.16
+django==4.2.17
 channels==4.1.0
 aiohttp==3.11.0
 django-cors-headers==4.6.0

From 9cfc4ec6dfd2facc51a39c329d8d09be90ee16e9 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 10 Dec 2024 09:42:45 +0800
Subject: [PATCH 105/200] fix(dryccfile): config not set

---
 rootfs/api/models/build.py      | 49 ++++++++++++++++++----------
 rootfs/api/models/config.py     | 58 +++++++++++++++++++++------------
 rootfs/api/tests/test_config.py | 35 ++++++++++++++++++--
 3 files changed, 103 insertions(+), 39 deletions(-)

diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 663ca4851..a460930c7 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -110,28 +110,43 @@ def _get_or_create_config(self):
         """
         dryccfile to config
         """
-        latest_release = Release.latest(self.app)
-        config_values, config_values_ref, config_healthcheck = [], {}, {}
-        for group, values in self.dryccfile.get('config', {}).items():
-            for value in values:
-                value['group'] = group
-                config_values.append(value)
+        config_values, config_values_ref, config_healthcheck, changed_fields = [], {}, {}, set()
+        if 'config' in self.dryccfile:
+            for group, values in self.dryccfile.get('config', {}).items():
+                for value in values:
+                    value['group'] = group
+                    config_values.append(value)
+            changed_fields.update(["values", "values_refs"])
+
+        replace_values_ptypes, replace_healthcheck_ptypes = set(), set()
         for ptype, values in self.dryccfile.get('deploy', {}).items():
-            for value in values.get('config', {}).get('env', []):
-                value['ptype'] = ptype
-                config_values.append(value)
-            for config_ref in values.get('config', {}).get('ref', []):
-                if ptype not in config_values_ref:
-                    config_values_ref[ptype] = [config_ref]
-                else:
-                    config_values_ref[ptype].append(config_ref)
+            if 'config' in values:
+                for value in values.get('config', {}).get('env', []):
+                    value['ptype'] = ptype
+                    config_values.append(value)
+                    replace_values_ptypes.add(ptype)
+                for config_ref in values.get('config', {}).get('ref', []):
+                    if ptype not in config_values_ref:
+                        config_values_ref[ptype] = [config_ref]
+                    else:
+                        config_values_ref[ptype].append(config_ref)
+                replace_values_ptypes.add(ptype)
+                changed_fields.update(["values", "values_refs"])
+
             if 'healthcheck' in values:
                 config_healthcheck[ptype] = values.get('healthcheck')
-        if not config_values:
-            return latest_release.config
+                changed_fields.add("healthcheck")
+                replace_healthcheck_ptypes.add(ptype)
+
+        old_config = self.app.release_set.filter(failed=False).latest().config
+        if not changed_fields:
+            return old_config
         config = Config(
             owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref,
             healthcheck=config_healthcheck,
         )
-        config.save(ignore_update_fields=["values", "values_refs", "healthcheck"])
+        config.merge_field("values", old_config, replace_values_ptypes)
+        config.merge_field("values_refs", old_config, replace_values_ptypes)
+        config.merge_field("healthcheck", old_config, replace_healthcheck_ptypes)
+        config.save(ignore_update_fields=changed_fields)
         return config
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 7b57ae068..7be9ed191 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -134,17 +134,23 @@ def save(self, ignore_update_fields=None, *args, **kwargs):
                 previous_config = self.app.config_set.latest()
             for field in self.allof_fields:
                 if ignore_update_fields is None or field not in ignore_update_fields:
-                    getattr(
-                        self,
-                        "_update_%s" % field,
-                        partial(self._update_field, field)
-                    )(previous_config)
+                    self.merge_field(field, previous_config)
         except Config.DoesNotExist:
             self._update_tags(previous_config={'tags': {}})
         return super(Config, self).save(*args, **kwargs)
 
-    def _update_field(self, field, previous_config):
-        data = getattr(previous_config, field, {}).copy()
+    def merge_field(self, field, old_config, replace_ptypes=[]):
+        getattr(
+            self,
+            "_update_%s" % field,
+            partial(self._update_field, field)
+        )(old_config, replace_ptypes)
+
+    def _update_field(self, field, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, field, {}).copy().items()
+            if k not in replace_ptypes
+        }
         new_data = getattr(self, field, {}).copy()
         # remove config keys if a null value is provided
         for key, value in new_data.items():
@@ -158,8 +164,11 @@ def _update_field(self, field, previous_config):
                 data[key] = self._merge_data(field, data.get(key, {}), value)
         setattr(self, field, data)
 
-    def _update_values(self, previous_config):
-        data = getattr(previous_config, 'values', []).copy()
+    def _update_values(self, previous_config, replace_ptypes=[]):
+        data = [
+            item for item in getattr(previous_config, 'values', []).copy()
+            if item.get('ptype') not in replace_ptypes
+        ]
         new_data = getattr(self, 'values', []).copy()
         for new_item in new_data:
             added = True
@@ -175,12 +184,15 @@ def _update_values(self, previous_config):
                     else:  # force to string
                         new_item['value'] = str(new_item['value'])
                     break
-            if added and new_item['value'] is not None and new_item['value'] != "":
+            if added and new_item['value'] is not None:
                 data.append(new_item)
         setattr(self, 'values', data)
 
-    def _update_values_refs(self, previous_config):
-        data = getattr(previous_config, 'values_refs', {}).copy()
+    def _update_values_refs(self, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, 'values_refs', {}).copy().items()
+            if k not in replace_ptypes
+        }
         new_data = getattr(self, 'values_refs', {}).copy()
         # remove config keys if a null value is provided
         for ptype, values in new_data.items():
@@ -192,7 +204,7 @@ def _update_values_refs(self, previous_config):
                 data.pop(ptype)
             else:
                 values_refs = data.get(ptype, [])
-                values_refs.extend(new_data.get(ptype, []))
+                values_refs.extend(values)
                 data[ptype] = list(set(values_refs))
         setattr(self, 'values_refs', data)
 
@@ -208,9 +220,12 @@ def _merge_data(self, field, data, new_data):
                 data[key] = value
         return data
 
-    def _update_tags(self, previous_config):
+    def _update_tags(self, previous_config, replace_ptypes=[]):
         """verify the tags exist on any nodes as labels"""
-        data = getattr(previous_config, 'tags', {}).copy()
+        data = {
+            k: v for k, v in getattr(previous_config, 'tags', {}).copy().items()
+            if k not in replace_ptypes
+        }
         new_data = getattr(self, 'tags', {}).copy()
         # remove config keys if a null value is provided
         for ptype, values in new_data.items():
@@ -236,14 +251,17 @@ def _update_tags(self, previous_config):
                     'tags', data.get(ptype, {}), values)
         setattr(self, 'tags', data)
 
-    def _update_limits(self, previous_config):
-        data = getattr(previous_config, 'limits', {}).copy()
+    def _update_limits(self, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, 'limits', {}).copy().items()
+            if k not in replace_ptypes
+        }
         new_data = getattr(self, 'limits', {}).copy()
         # check procfile
-        for key, value in new_data.items():
+        for ptype, value in new_data.items():
             if value is None:
-                if key in self.app.ptypes:
+                if ptype in self.app.ptypes:
                     raise UnprocessableEntity(
-                        "the %s has already been used and cannot be deleted" % key)
+                        "the %s has already been used and cannot be deleted" % ptype)
         self._merge_data('limits', data, new_data)
         setattr(self, 'limits', data)
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index eda4ac74d..fe196448f 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -4,6 +4,7 @@
 
 Run the tests with "./manage.py test api"
 """
+import copy
 from io import StringIO
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
@@ -829,7 +830,7 @@ def test_config_from_dryccfile(self, mock_requests):
                 'deploy': {
                     'web': {
                         'image': "127.0.0.1/cat/cat"
-                    }
+                    },
                 }
             },
         }
@@ -876,7 +877,7 @@ def test_config_from_dryccfile(self, mock_requests):
             release.config.envs("web"),
             {
                 'GROUP': 'g1', 'DEBUG': 'tr', 'TEST1': 'g1', 'TEST2': 'tr', 'PENV1': 'web',
-                'PENV2': 'web'
+                'PENV2': 'web', "WEBSITE": "www.drycc.cc",
             }
         )
         build_body['dryccfile']['deploy']['web']['healthcheck'] = {
@@ -901,3 +902,33 @@ def test_config_from_dryccfile(self, mock_requests):
             release.config.healthcheck['web'],
             build_body['dryccfile']['deploy']['web']['healthcheck'],
         )
+        # test use old config and healthcheck
+        new_build_body = copy.deepcopy(build_body)
+        del new_build_body['dryccfile']['config']
+        del new_build_body['dryccfile']['deploy']['web']['config']
+        del new_build_body['dryccfile']['deploy']['web']['healthcheck']
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, new_build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(release.config.healthcheck, release.previous().config.healthcheck)
+        self.assertEqual(release.config.values, release.previous().config.values)
+        self.assertEqual(release.config.values_refs, release.previous().config.values_refs)
+        # set empty
+        new_build_body = copy.deepcopy(build_body)
+        new_build_body['dryccfile']['config'] = {}
+        new_build_body['dryccfile']['deploy']['web']['config'] = {}
+        new_build_body['dryccfile']['deploy']['web']['healthcheck'] = {}
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, new_build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(release.config.healthcheck, {'web': {}})
+        self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
+        self.assertEqual(release.config.values_refs, {})

From 9b0cc0a9a658c3f3390a13eb2ea979773b21f2da Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Fri, 13 Dec 2024 17:33:54 +0800
Subject: [PATCH 106/200] chore(controller): request check name parameter
 (#174)

* chore(controller): mount should not release

* fix(controller): app destroy error

* chore(controller): support image pull secrets by ptype

* chore(controller): list pod when pending

* chore(controller): modify clean ptype check

* chore(controller): request check name parameter
---
 charts/controller/templates/_helpers.tpl      |  7 +++
 .../templates/controller-cronjob-daily.yaml   |  6 ++
 .../templates/controller-cronjob-hourly.yaml  |  5 +-
 .../templates/controller-job-upgrade.yaml     |  2 +
 rootfs/api/models/app.py                      | 58 ++++++++++++-------
 rootfs/api/models/gateway.py                  |  1 +
 rootfs/api/serializers/__init__.py            | 24 +++++++-
 rootfs/api/settings/production.py             |  8 ++-
 rootfs/api/tests/test_app.py                  | 16 ++---
 rootfs/api/tests/test_app_settings.py         |  6 +-
 rootfs/api/tests/test_build.py                | 25 +++++++-
 rootfs/api/tests/test_gateway.py              | 57 ++++++++++++++++++
 rootfs/api/tests/test_resource.py             | 20 +++++++
 rootfs/api/tests/test_volume.py               | 32 ++++++++++
 rootfs/api/urls.py                            | 45 ++++++++------
 rootfs/api/views.py                           |  9 ++-
 16 files changed, 265 insertions(+), 56 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 42b9979a2..6a0af9007 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -177,6 +177,13 @@ env:
 {{- end }}
 {{- end }}
 
+
+{{- define "controller-job.envs" }}
+env:
+- name: DRYCC_DATABASE_ROUTERS
+  value: api.routers.DefaultReplicaRouter
+{{- end }}
+
 {{/* Generate controller deployment limits */}}
 {{- define "controller.limits" -}}
 {{- if or (.Values.limitsCpu) (.Values.limitsMemory) }}
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index 365149f9d..9265732df 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -33,6 +33,7 @@ spec:
               - -a
               - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
@@ -47,6 +48,7 @@ spec:
             - python -u /workspace/manage.py clearsocial
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.imagePullPolicy}}
             name: drycc-controller-load-db-state-to-k8s
@@ -60,6 +62,7 @@ spec:
               - python /workspace/manage.py load_db_state_to_k8s
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-measure-apps
@@ -73,6 +76,7 @@ spec:
               - python -u /workspace/manage.py measure_apps
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-measure-resources
@@ -86,6 +90,7 @@ spec:
             - python -u /workspace/manage.py measure_resources
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-measure-volumes
@@ -99,3 +104,4 @@ spec:
             - python -u /workspace/manage.py measure_volumes
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 4bec8c2d4..3fa6f22f4 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -33,6 +33,7 @@ spec:
               - -a
               - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
@@ -47,6 +48,7 @@ spec:
             - python -u /workspace/manage.py measure_networks
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-measure-loadbalancer
@@ -59,4 +61,5 @@ spec:
             - -c
             - python -u /workspace/manage.py measure_loadbalancers
             {{- end }}
-            {{- include "controller.envs" . | indent 12 }}
\ No newline at end of file
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
\ No newline at end of file
diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index af78741e2..d01031528 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -20,6 +20,7 @@ spec:
         - -u
         - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         {{- include "controller.envs" . | indent 8 }}
+        {{- include "controller-job.envs" . | indent 12 }}
       containers:
       - name: drycc-controller-job-upgrade
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
@@ -38,6 +39,7 @@ spec:
           python -u /workspace/manage.py loaddata /etc/controller/limit-plans.json
         {{- end }}
         {{- include "controller.envs" . | indent 8 }}
+        {{- include "controller-job.envs" . | indent 12 }}
         volumeMounts:
         - name: controller-config
           readOnly: false
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 429304f6c..e9c4ec9a8 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -270,7 +270,7 @@ def restart(self, **kwargs):  # noqa
     def scale(self, user, structure):
         err_msg = None
         release = Release.latest(self)
-        if (PTYPE_RUN in structure or release.build is None):
+        if (PTYPE_RUN in structure or release is None or release.build is None):
             if PTYPE_RUN in structure:
                 err_msg = 'Cannot set scale for reserved types, procfile type is: run'
             else:
@@ -283,7 +283,7 @@ def scale(self, user, structure):
     def pipeline(self, release, ptypes, force_deploy=False):
         prefix = f"[pipeline] release {release.version_name}"
         try:
-            if release.build is not None:
+            if release is not None and release.build is not None:
                 if release.build.dryccfile:
                     for run in release.get_runners(ptypes):
                         self.log(f"{prefix} starts running pipeline.run {run['image']}")
@@ -323,7 +323,7 @@ def deploy(self, release, ptypes=None, force_deploy=False, rollback_on_failure=T
 
         force_deploy can be used when a deployment is broken, such as for Rollback
         """
-        if release.build is None:
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release')
         # use create to make sure minimum resources are created
         self.create()
@@ -352,7 +352,7 @@ def mount(self, user, volume, structure=None):
         if release is None or release.build is None:
             raise DryccException('No build associated with this release')
         app_settings = self.appsettings_set.latest()
-        self._mount(user, volume, release, app_settings, structure=structure)
+        self._mount(user, volume, app_settings, structure=structure)
 
     def clean(self, release=None, ptypes=None):
         release = release if release else self.release_set.latest()
@@ -431,17 +431,21 @@ def get_command_and_args(pod, container_name):
         result = []
         try:
             pod = self.scheduler().pod.get(self.id, pod_name).json()
-            for status in pod["status"]["containerStatuses"]:
+            if pod["status"]['phase'] == 'Pending':
+                statuses = pod["spec"]["containers"]
+            else:
+                statuses = pod["status"]["containerStatuses"]
+            for status in statuses:
                 command, args = get_command_and_args(pod, status["name"])
                 result.append({
                     "container": status["name"],
                     "image": status["image"],
                     "command": command,
                     "args": args,
-                    "state": status["state"],
-                    "lastState": status["lastState"],
-                    "ready": status["ready"],
-                    "restartCount": status["restartCount"],
+                    "state": status.get("state", {}),
+                    "lastState": status.get("lastState", {}),
+                    "ready": status.get("ready", False),
+                    "restartCount": status.get("restartCount", 0),
                     "status": pod["status"].get("phase", ""),
                     "reason": pod["status"].get("reason", ""),
                     "message": pod["status"].get("message", "")
@@ -470,15 +474,22 @@ def list_pods(self, *args, **kwargs):
                 else:
                     started = str(
                         datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT))
+                state = str(self.scheduler().pod.state(p))
+                if p['status']['phase'] != 'Pending':
+                    ready = len([1 for s in p["status"]["containerStatuses"] if s['ready']])
+                    restarts = sum([s['restartCount'] for s in p["status"]["containerStatuses"]])
+                else:
+                    restarts = 0
+                    ready = 0
                 item = {
-                    'name': p['metadata']['name'], 'state': str(self.scheduler().pod.state(p)),
+                    'name': p['metadata']['name'],
+                    'state': state,
                     'release': labels['version'], 'type': labels['type'], 'started': started,
                     'ready': "%s/%s" % (
-                        len([1 for s in p["status"]["containerStatuses"] if s['ready']]),
-                        len(p["status"]["containerStatuses"]),
+                        ready,
+                        len(p["spec"]["containers"]),
                     ),
-                    'restarts': sum(
-                        [s['restartCount'] for s in p["status"]["containerStatuses"]]),
+                    'restarts': restarts
                 }
                 data.append(item)
             # sorting so latest start date is first
@@ -626,12 +637,12 @@ def autoscale(self, proc_type, autoscale):
                 # let the user know about any other errors
                 raise ServiceUnavailable(str(e)) from e
 
-    def image_pull_secret(self, namespace, registry, image):
+    def image_pull_secret(self, namespace, ptype, registry, image):
         """
         Take registry information and set as an imagePullSecret for an RC / Deployment
         http://kubernetes.io/docs/user-guide/images/#specifying-imagepullsecrets-on-a-pod
         """
-        docker_config, name, create = self._get_private_registry_config(image, registry)
+        docker_config, name, create = self._get_private_registry_config(ptype, image, registry)
         if create is None:
             return
         elif create:
@@ -741,11 +752,14 @@ def _clean_app_logs(self):
             err = 'Error deleting existing application logs: {}'.format(e)
             self.log(err, logging.WARNING)
 
-    def _mount(self, user, volume, release, app_settings, structure=None):
+    def _mount(self, user, volume, app_settings, structure=None):
         volumes = Volume.objects.filter(app=self)
         tasks = []
         for scale_type, replicas in structure.items() if structure else self.structure.items():
             if scale_type != PTYPE_RUN:
+                release = self.release_set.filter(
+                    deployed_ptypes__contains=scale_type,
+                    failed=False).latest()
                 replicas = self.structure.get(scale_type, 0)
                 scale_type_volumes = [
                     volume for volume in volumes if scale_type in volume.path.keys()]
@@ -810,7 +824,7 @@ def _deploy(self, deploys, ptypes, prev_release,
             except KubeException as e:
                 # Don't rollback if the previous release doesn't have a build which means
                 # this is the first build and all the previous releases are just config changes.
-                if (rollback_on_failure and prev_release.build is not None):
+                if rollback_on_failure and prev_release is not None and prev_release.build is not None:  # noqa
                     err = 'There was a problem deploying {}. Rolling back to release {}.'.format(
                         release.version_name, prev_release.version_name)
                     # This goes in the log before the rollback starts
@@ -1098,7 +1112,7 @@ def _build_env_vars(self, release, ptype):
         Build a dict of env vars, setting default vars based on app type
         and then combining with the user set ones
         """
-        if release.build is None:
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release to run this command')
 
         # mix in default environment information drycc may require
@@ -1119,8 +1133,8 @@ def _build_env_vars(self, release, ptype):
             default_env['PORT'] = port
         return default_env
 
-    def _get_private_registry_config(self, image, registry=None):
-        name = settings.REGISTRY_SECRET_PREFIX
+    def _get_private_registry_config(self, ptype, image, registry=None):
+        name = settings.REGISTRY_SECRET_PREFIX + '-' + ptype
         if registry:
             # try to get the hostname information
             hostname = registry.get('hostname', None)
@@ -1206,7 +1220,7 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
         registry = config.registry.get(ptype, {})
         # create image pull secret if needed
         image_pull_secret_name = self.image_pull_secret(
-            self.id, registry, release.get_deploy_image(ptype))
+            self.id, ptype, registry, release.get_deploy_image(ptype))
 
         # only web is routable
         # https://www.drycc.cc/applications/managing-app-processes/#default-process-types
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 9312f63b3..af1cad24e 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -11,6 +11,7 @@
 
 from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT
 
+
 User = get_user_model()
 logger = logging.getLogger(__name__)
 
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index ad9a0fe23..60964aff8 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -80,6 +80,19 @@ def validate_ptype(value):
     return value
 
 
+def validate_name(value):
+    NAME_REGEX = r'^(?P<name>{})$'.format(settings.NAME_REGEX)
+    NAME_MATCH = re.compile(NAME_REGEX)
+    NAME_MISMATCH_MSG = "name can only supports: %s" % NAME_MATCH.pattern
+    if not re.match(NAME_REGEX, value):
+        raise serializers.ValidationError(NAME_MISMATCH_MSG)
+    if len(value) < PTYPE_MIN_LENGTH or len(value) > PTYPE_MAX_LENGTH:
+        raise serializers.ValidationError(
+            "The length of name must be between {} and {}".format(
+                PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH))
+    return value
+
+
 class JSONFieldSerializer(serializers.JSONField):
     def __init__(self, *args, **kwargs):
         self.convert_to_str = kwargs.pop('convert_to_str', True)
@@ -585,7 +598,8 @@ class Meta:
 
     @staticmethod
     def validate_autoscale(data):
-        for _, autoscale in data.items():
+        for ptype, autoscale in data.items():
+            validate_ptype(ptype)
             if autoscale is None:
                 continue
             validate_json(autoscale, AUTOSCALE_SCHEMA, serializers.ValidationError)
@@ -634,6 +648,8 @@ def validate_size(self, data):
             raise serializers.ValidationError(VOLUME_SIZE_MISMATCH_MSG)
         return data.upper()
 
+    validate_name = staticmethod(validate_name)
+
     @staticmethod
     def validate_path(data):
         logger.debug(f"mount validate_path data: {data}")
@@ -691,6 +707,8 @@ def update(self, instance, validated_data):
         instance.save()
         return instance
 
+    validate_name = staticmethod(validate_name)
+
 
 class MetricSerializer(serializers.Serializer):
     start = serializers.IntegerField(
@@ -733,6 +751,8 @@ def to_representation(self, instance):
         representation['addresses'] = instance.addresses
         return representation
 
+    validate_name = staticmethod(validate_name)
+
     @staticmethod
     def validate_protocol(value):
         if not re.match(GATEWAY_PROTOCOL_MATCH, value):
@@ -756,6 +776,8 @@ class Meta:
         model = models.gateway.Route
         fields = '__all__'
 
+    validate_name = staticmethod(validate_name)
+
     @staticmethod
     def validate_kind(value):
         if not re.match(ROUTE_PROTOCOL_MATCH, value):
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 876c90c30..d8ce0c0e8 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -189,7 +189,7 @@
             'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
         },
         'simple': {
-            'format': '%(levelname)s %(message)s'
+            'format': '%(levelname)s %(asctime)s %(message)s'
         },
     },
     'filters': {
@@ -372,12 +372,16 @@
     DATABASES["replica"] = dj_database_url.config(default=DRYCC_DATABASE_REPLICA_URL)
 
 # database routers
-DATABASE_ROUTERS = ['api.routers.DefaultReplicaRouter', ]
+# Implements: 'api.routers.DefaultReplicaRouter'
+DATABASE_ROUTERS = [r for r in os.environ.get('DRYCC_DATABASE_ROUTERS', '').split(',') if r]
+
 
 APP_URL_REGEX = '[a-z0-9-]+'
 
 DOMAIN_URL_REGEX = r'\**\.?[-\._\w]+'
 
+NAME_REGEX = r'[a-z0-9]+(\-[a-z0-9]+)*'
+
 # Oauth settings
 
 DRYCC_PASSPORT_URL = os.environ.get('DRYCC_PASSPORT_URL', 'https://127.0.0.1:8000')
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 370f0537a..77ffa6aed 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -543,24 +543,24 @@ def test_list_ordering(self, mock_requests):
         self.assertEqual(apps[3]['id'], 'zulu')
 
     def test_get_private_registry_config(self, mock_requests):
-        registry = {'username': 'test', 'password': 'test'}
+        registry = {"web": {'username': 'test', 'password': 'test'}}
         auth = bytes('{}:{}'.format("test", "test"), 'UTF-8')
         encAuth = base64.b64encode(auth).decode(encoding='UTF-8')
         image = 'test/test'
 
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry.get("web", {}))  # noqa
         dockerConfig = json.loads(docker_config)
         expected = {"https://index.docker.io/v1/": {"auth": encAuth}}
         self.assertEqual(dockerConfig.get('auths'), expected)
-        self.assertEqual(name, "private-registry")
+        self.assertEqual(name, "private-registry-web")
         self.assertEqual(create, True)
 
         image = "quay.io/test/test"
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry.get("web", {}))  # noqa
         dockerConfig = json.loads(docker_config)
         expected = {"quay.io": {"auth": encAuth}}
         self.assertEqual(dockerConfig.get('auths'), expected)
-        self.assertEqual(name, "private-registry")
+        self.assertEqual(name, "private-registry-web")
         self.assertEqual(create, True)
 
     @override_settings(REGISTRY_LOCATION="off-cluster")
@@ -569,20 +569,20 @@ def test_get_private_registry_config_off_cluster(self, mock_requests):
         auth = bytes('{}:{}'.format("test", "test"), 'UTF-8')
         encAuth = base64.b64encode(auth).decode(encoding='UTF-8')
         image = "test.com/test/test"
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry.get("web", {}))  # noqa
         dockerConfig = json.loads(docker_config)
         expected = {"https://index.docker.io/v1/": {
             "auth": encAuth
         }}
         self.assertEqual(dockerConfig.get('auths'), expected)
-        self.assertEqual(name, "private-registry-off-cluster")
+        self.assertEqual(name, "private-registry-web-off-cluster")
         self.assertEqual(create, True)
 
     @override_settings(REGISTRY_LOCATION="ecra")
     def test_get_private_registry_config_bad_registry_location(self, mock_requests):
         registry = {}
         image = "test.com/test/test"
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry)
         self.assertEqual(docker_config, None)
         self.assertEqual(name, None)
         self.assertEqual(create, None)
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
index 92cb3a9a2..794d77004 100644
--- a/rootfs/api/tests/test_app_settings.py
+++ b/rootfs/api/tests/test_app_settings.py
@@ -136,8 +136,12 @@ def test_autoscale(self, mock_requests):
         # check that config fails if trying to unset non-existing proc type
         response = self.client.post(
             f'/v2/apps/{app_id}/settings',
-            {'autoscale': {'invalid_proctype': None}})
+            {'autoscale': {'invalid-proctype': None}})
         self.assertEqual(response.status_code, 422, response.data)
+        response = self.client.post(
+            f'/v2/apps/{app_id}/settings',
+            {'autoscale': {'invalid_proctype': None}})
+        self.assertEqual(response.status_code, 400, response.data)
 
         # remove a proc type
         response = self.client.post(
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index a7ba312b1..515fdb488 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -351,7 +351,17 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.json()['structure'], {'web': 1, 'task': 1, 'worker': 1})
 
-        # clean worker
+        # clean worker, repicas 1 not allow
+        response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id), {"ptypes": "task"})
+        self.assertEqual(response.status_code, 400, response.data)
+
+        # scale task
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'task': 0}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # clean worker, repicas 0 allow
         response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id), {"ptypes": "task"})
         self.assertEqual(response.status_code, 204, response.data)
 
@@ -361,8 +371,19 @@ def test_build_no_remove_process(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.json()['structure'], {'web': 1, 'worker': 1})
 
-        # clean all worker
+        # ptypes is a required field
         response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id))
+        self.assertEqual(response.status_code, 400, response.data)
+
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'worker': 0}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # clean worker
+        response = self.client.post(
+            '/v2/apps/{}/ptypes/clean'.format(app_id),
+            {"ptypes": "worker"})
         self.assertEqual(response.status_code, 204, response.data)
 
         # look at the app structure
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index f0904fea6..1bd42d31f 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -71,6 +71,27 @@ def create_gateway(self, app_id, name, port, protocol):
         )
         self.assertEqual(response.status_code, 201)
 
+    def create_gateway_name(self):
+        app_id = self.create_app()
+        name1 = 'gatway_'
+        response = self.client.post(
+            f'/v2/apps/{app_id}/gateways/',
+            {'name': name1, 'port': 80, 'protocol': 'HTTP'}
+        )
+        self.assertEqual(response.status_code, 400)
+        name2 = '-gatway'
+        response = self.client.post(
+            f'/v2/apps/{app_id}/gateways/',
+            {'name': name2, 'port': 80, 'protocol': 'HTTP'}
+        )
+        self.assertEqual(response.status_code, 400)
+        name3 = 'test.gatway'
+        response = self.client.post(
+            f'/v2/apps/{app_id}/gateways/',
+            {'name': name3, 'port': 80, 'protocol': 'HTTP'}
+        )
+        self.assertEqual(response.status_code, 400)
+
     def create_tls_domain(self, app_id):
         cert_url = f'/v2/apps/{app_id}/certs'
         secret_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(23))
@@ -463,6 +484,42 @@ def test_create_route(self):
         self.assertEqual(response.data["count"], 1)
         self.assertEqual(len(response.data["results"][0]["rules"]), 1)
 
+        # name regex format
+        name1 = "test-route_"
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "port": 5000,
+                "ptype": "no-exists",
+                "kind": "HTTPRoute",
+                "name": name1,
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+        # name regex format
+        name2 = "-test-route"
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "port": 5000,
+                "ptype": "no-exists",
+                "kind": "HTTPRoute",
+                "name": name2,
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+        name3 = "test.route"
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "port": 5000,
+                "ptype": "no-exists",
+                "kind": "HTTPRoute",
+                "name": name3,
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+
     def test_hostnames(self):
         app_id = self.create_app()
         ptype, port, route_name = self.create_route(app_id)
diff --git a/rootfs/api/tests/test_resource.py b/rootfs/api/tests/test_resource.py
index 3d6590bed..1c591d9d0 100644
--- a/rootfs/api/tests/test_resource.py
+++ b/rootfs/api/tests/test_resource.py
@@ -33,6 +33,26 @@ def test_resources_create(self, mock_requests):
         """Test that the serialized response contains only relevant data."""
         app_id = self.create_app()
 
+        # name regex error
+        name1 = 'mysql_'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': name1, 'plan': 'mysql:5.6'}
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name2 = '-mysql'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': name2, 'plan': 'mysql:5.6'}
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name3 = 'mysql.8'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': name3, 'plan': 'mysql:5.6'}
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+
         response = self.client.post(
             '/v2/apps/{}/resources'.format(app_id),
             data={'name': 'mysql', 'plan': 'mysql:5.6'}
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index e3d180593..d95301a59 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -51,6 +51,38 @@ def test_volumecreate(self, mock_requests):
         )
         self.assertEqual(response.status_code, 400, response.data)
 
+        # name regex error
+        name1 = 'myvolume_'
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': name1,
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name2 = '-myvolume'
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': name2,
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name3 = 'my.volume'
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': name3,
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+
         response = self.client.post(
             '/v2/apps/{}/volumes'.format(app_id),
             data={
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index ddf3cfa6b..698722b9d 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -51,7 +51,8 @@
         views.PodViewSet.as_view({'get': 'list', 'delete': 'delete'})),
     # describe pod
     re_path(
-        r"^apps/(?P<id>{})/pods/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/pods/(?P<name>{})/describe/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.PodViewSet.as_view({'get': 'describe'})),
     # restart deployment/ptype's pods
     re_path(
@@ -71,7 +72,8 @@
         views.PtypeViewSet.as_view({'get': 'list'})),
     # describe ptypes
     re_path(
-        r"^apps/(?P<id>{})/ptypes/(?P<name>[-_\w]+)/describe/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/ptypes/(?P<name>{})/describe/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.PtypeViewSet.as_view({'get': 'describe'})),
     # list events
     re_path(
@@ -107,20 +109,22 @@
         r"^apps/(?P<id>{})/volumes/?$".format(settings.APP_URL_REGEX),
         views.AppVolumesViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/volumes/(?P<name>{})/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppVolumesViewSet.as_view(
             {'get': 'retrieve', 'patch': 'expand', 'delete': 'destroy'})),
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/path/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/volumes/(?P<name>{})/path/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppVolumesViewSet.as_view({'patch': 'path'})),
     # application filer
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/client/?$".format(
-            settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/volumes/(?P<name>{})/client/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppFilerClientViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/client/(?P<path>[\S]+)$".format(
-            settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/volumes/(?P<name>{})/client/(?P<path>[\S]+)$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppFilerClientViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
     # application resources
     re_path(r"^resources/services/?$", views.AppResourcesViewSet.as_view({'get': 'services'})),
@@ -131,19 +135,22 @@
         r"^apps/(?P<id>{})/resources/?$".format(settings.APP_URL_REGEX),
         views.AppResourcesViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
-        r"^apps/(?P<id>{})/resources/(?P<name>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/resources/(?P<name>{})/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppSingleResourceViewSet.as_view(
             {'get': 'retrieve', 'delete': 'destroy', 'put': 'update'})),
     re_path(
-        r"^apps/(?P<id>{})/resources/(?P<name>[-_\w]+)/binding/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/resources/(?P<name>{})/binding/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppResourceBindingViewSet.as_view({'patch': 'binding'})),
     # certificates
     re_path(
-        r'^apps/(?P<id>{})/certs/(?P<name>[-_*.\w]+)/domain/(?P<domain>{})?/?$'.format(
-            settings.APP_URL_REGEX, settings.DOMAIN_URL_REGEX),
+        r'^apps/(?P<id>{})/certs/(?P<name>{})/domain/(?P<domain>{})?/?$'.format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX, settings.DOMAIN_URL_REGEX),
         views.CertificateViewSet.as_view({'delete': 'detach', 'post': 'attach'})),
     re_path(
-        r'^apps/(?P<id>{})/certs/(?P<name>[-_*.\w]+)/?$'.format(settings.APP_URL_REGEX),
+        r'^apps/(?P<id>{})/certs/(?P<name>{})/?$'.format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.CertificateViewSet.as_view({
             'get': 'retrieve',
             'delete': 'destroy'
@@ -192,17 +199,21 @@
             {'post': 'create_or_update', 'get': 'list', 'delete': 'delete'})),
     # routes
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)?/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})?/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view(
             {'post': 'create', 'get': 'list', 'delete': 'delete'})),
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)/attach/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})/attach/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'patch': 'attach'})),
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)/detach/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})/detach/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'patch': 'detach'})),
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)/rules/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})/rules/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'get': 'get', 'put': 'set'})),
     # users
     re_path(r'^users/?$', views.UserView.as_view({'get': 'list'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 88720daca..9e4681cbd 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -322,7 +322,7 @@ def run(self, request, **kwargs):
         if not command:
             raise DryccException('command is a required field, or it can be defined in Procfile')
         release = models.release.Release.latest(app)
-        if release.build is None:
+        if release is None or release.build is None:
             raise DryccException('no build available, please deploy a release')
         volumes = request.data.get('volumes', None)
         if volumes:
@@ -491,7 +491,12 @@ def clean(self, request, *args, **kwargs):
         app = self.get_app()
         ptypes = set(
             [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
-        ptypes = app.check_ptypes(ptypes)
+        if not ptypes:
+            raise DryccException("ptypes is a required field")
+        latest_ptypes = [k for k, v in app.structure.items() if v != 0]
+        not_allow = [ptype for ptype in ptypes if ptype in latest_ptypes]
+        if not_allow:
+            raise DryccException(f'ptype {",".join(not_allow)} should not garbage.')
         app.clean(ptypes=ptypes)
         return Response(status=status.HTTP_204_NO_CONTENT)
 

From 4083a16939756307cade6938cae9e68f151c93aa Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 23 Dec 2024 09:40:06 +0800
Subject: [PATCH 107/200] fix(wooddpecker): secsets are deprecated

---
 .woodpecker/build-linux.yml | 17 +++++++++++------
 .woodpecker/chart.yaml      | 14 +++++++++-----
 .woodpecker/manifest.yml    | 13 ++++++++-----
 .woodpecker/test-linux.yml  | 11 +++++++----
 4 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml
index 736b01e89..e9d18af8d 100644
--- a/.woodpecker/build-linux.yml
+++ b/.woodpecker/build-linux.yml
@@ -14,12 +14,17 @@ steps:
   - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM)
   - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1
   - make podman-build podman-immutable-push
-  secrets:
-  - codename
-  - dev_registry
-  - drycc_registry
-  - container_username
-  - container_password
+  environment:
+    CODENAME:
+      from_secret: codename
+    DEV_REGISTRY:
+      from_secret: dev_registry
+    DRYCC_REGISTRY:
+      from_secret: drycc_registry
+    CONTAINER_USERNAME:
+      from_secret: container_username
+    CONTAINER_PASSWORD:
+      from_secret: container_password
   when:
     event:
     - push
diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml
index 5aa7eb34e..40397e008 100644
--- a/.woodpecker/chart.yaml
+++ b/.woodpecker/chart.yaml
@@ -14,11 +14,15 @@ steps:
   - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION
   - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin
   - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts)
-  secrets:
-  - dev_registry
-  - drycc_registry
-  - container_username
-  - container_password
+  environment:
+    DEV_REGISTRY:
+      from_secret: dev_registry
+    DRYCC_REGISTRY:
+      from_secret: drycc_registry
+    CONTAINER_USERNAME:
+      from_secret: container_username
+    CONTAINER_PASSWORD:
+      from_secret: container_password
   when:
     event:
     - push
diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml
index 052f5d803..d1f448393 100644
--- a/.woodpecker/manifest.yml
+++ b/.woodpecker/manifest.yml
@@ -8,8 +8,9 @@ steps:
   commands:
   - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl
   - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl
-  secrets:
-  - drycc_registry
+  environment:
+    DRYCC_REGISTRY:
+      from_secret: drycc_registry
   when:
     event:
     - tag
@@ -26,9 +27,11 @@ steps:
     -v $(pwd):$(pwd)
     -w $(pwd)
     plugins/manifest
-  secrets:
-  - container_username
-  - container_password
+  environment:
+    CONTAINER_USERNAME:
+      from_secret: container_username
+    CONTAINER_PASSWORD:
+      from_secret: container_password
   when:
     event:
     - tag
diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml
index 373868083..ded271221 100644
--- a/.woodpecker/test-linux.yml
+++ b/.woodpecker/test-linux.yml
@@ -12,10 +12,13 @@ steps:
   image: bash
   commands:
   - make test podman-build-test upload-coverage
-  secrets:
-  - codename
-  - dev_registry
-  - codecov_token
+  environment:
+    CODENAME:
+      from_secret: codename
+    DEV_REGISTRY:
+      from_secret: dev_registry
+    CODECOV_TOKEN:
+      from_secret: codecov_token
   when:
     event:
     - push

From 7cb6bfaca410f28bc4aa236505e18782b8663d8f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 24 Dec 2024 17:03:03 +0800
Subject: [PATCH 108/200] fix(charts): indentation error

---
 charts/controller/templates/_helpers.tpl                   | 2 +-
 charts/controller/templates/controller-cronjob-daily.yaml  | 1 -
 charts/controller/templates/controller-cronjob-hourly.yaml | 1 -
 charts/controller/templates/controller-job-upgrade.yaml    | 2 --
 4 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 6a0af9007..9145dce3c 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -179,7 +179,7 @@ env:
 
 
 {{- define "controller-job.envs" }}
-env:
+{{- include "controller.envs" . }}
 - name: DRYCC_DATABASE_ROUTERS
   value: api.routers.DefaultReplicaRouter
 {{- end }}
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index 9265732df..5766e0290 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -32,7 +32,6 @@ spec:
               - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
               - -a
               - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
-            {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 3fa6f22f4..0760503da 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -32,7 +32,6 @@ spec:
               - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
               - -a
               - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
-            {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index d01031528..af78741e2 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -20,7 +20,6 @@ spec:
         - -u
         - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
         {{- include "controller.envs" . | indent 8 }}
-        {{- include "controller-job.envs" . | indent 12 }}
       containers:
       - name: drycc-controller-job-upgrade
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
@@ -39,7 +38,6 @@ spec:
           python -u /workspace/manage.py loaddata /etc/controller/limit-plans.json
         {{- end }}
         {{- include "controller.envs" . | indent 8 }}
-        {{- include "controller-job.envs" . | indent 12 }}
         volumeMounts:
         - name: controller-config
           readOnly: false

From ce328423061b4edc36a99dd54a2705cdb5b751b0 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 25 Dec 2024 12:56:35 +0800
Subject: [PATCH 109/200] feat(limits): add runtime class name to plans

---
 charts/controller/templates/_helpers.tpl      |   5 +-
 .../0021_limitplan_runtime_class_name.py      |  18 +++
 rootfs/api/models/app.py                      |   5 +-
 rootfs/api/models/build.py                    |   9 +-
 rootfs/api/models/limit.py                    |   1 +
 rootfs/api/models/release.py                  |  22 ++--
 rootfs/api/serializers/schemas/dryccfile.py   |  84 ++++++++++++--
 rootfs/api/settings/production.py             |   2 -
 rootfs/api/tests/test_build.py                |  30 ++---
 rootfs/api/tests/test_config.py               |  23 ++--
 rootfs/api/tests/test_release.py              | 106 +++++++++---------
 11 files changed, 188 insertions(+), 117 deletions(-)
 create mode 100644 rootfs/api/migrations/0021_limitplan_runtime_class_name.py

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 9145dce3c..c297db592 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -26,10 +26,6 @@ env:
 - name: "DRYCC_APP_STORAGE_CLASS"
   value: "{{ (tpl .Values.appStorageClass .) }}"
 {{- end }}
-{{- if (.Values.appRuntimeClass) }}
-- name: "DRYCC_APP_RUNTIME_CLASS"
-  value: "{{ .Values.appRuntimeClass }}"
-{{- end }}
 {{- if (.Values.appDNSPolicy) }}
 - name: "DRYCC_APP_DNS_POLICY"
   value: "{{ .Values.appDNSPolicy }}"
@@ -260,6 +256,7 @@ resources:
       kubernetes.io/egress-bandwidth: 100M
       kubernetes.io/ingress-bandwidth: 100M
     node_selector: {}
+    runtime_class_name: ""
     pod_security_context: {}
     container_security_context: {}
     created: {{ now | date "2006-01-02T15:04:05.000Z" }}
diff --git a/rootfs/api/migrations/0021_limitplan_runtime_class_name.py b/rootfs/api/migrations/0021_limitplan_runtime_class_name.py
new file mode 100644
index 000000000..10e334f95
--- /dev/null
+++ b/rootfs/api/migrations/0021_limitplan_runtime_class_name.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.17 on 2024-12-25 08:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0020_remove_config_typed_values_alter_config_values'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='limitplan',
+            name='runtime_class_name',
+            field=models.CharField(default='', max_length=63),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index e9c4ec9a8..e1e702d44 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -289,7 +289,8 @@ def pipeline(self, release, ptypes, force_deploy=False):
                         self.log(f"{prefix} starts running pipeline.run {run['image']}")
                         job_name = self.run(
                             self.owner, run['image'], command=run['command'],
-                            args=run['args'], timeout=run['timeout'], expires=run['timeout']
+                            args=run['args'], timeout=run['timeout'], expires=run['timeout'],
+                            envs=self._build_env_vars(release, run['ptype']),
                         )
                         state, labels = 'initializing', {'job-name': job_name}
                         for count, state in enumerate(self.scheduler().pod.watch(
@@ -1240,7 +1241,7 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
             'build_type': release.build.type,
             'annotations': limit_plan.annotations,
             'healthcheck': healthcheck,
-            'runtime_class_name': settings.DRYCC_APP_RUNTIME_CLASS,
+            'runtime_class_name': limit_plan.runtime_class_name,
             'dns_policy': settings.DRYCC_APP_DNS_POLICY,
             'lifecycle_post_start': config.lifecycle_post_start,
             'lifecycle_pre_stop': config.lifecycle_pre_stop,
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index a460930c7..810ed90ea 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -110,14 +110,10 @@ def _get_or_create_config(self):
         """
         dryccfile to config
         """
-        config_values, config_values_ref, config_healthcheck, changed_fields = [], {}, {}, set()
+        config_values, config_values_ref, config_healthcheck, changed_fields = (
+            self.dryccfile.get('config', []), {}, {}, set())
         if 'config' in self.dryccfile:
-            for group, values in self.dryccfile.get('config', {}).items():
-                for value in values:
-                    value['group'] = group
-                    config_values.append(value)
             changed_fields.update(["values", "values_refs"])
-
         replace_values_ptypes, replace_healthcheck_ptypes = set(), set()
         for ptype, values in self.dryccfile.get('deploy', {}).items():
             if 'config' in values:
@@ -132,7 +128,6 @@ def _get_or_create_config(self):
                         config_values_ref[ptype].append(config_ref)
                 replace_values_ptypes.add(ptype)
                 changed_fields.update(["values", "values_refs"])
-
             if 'healthcheck' in values:
                 config_healthcheck[ptype] = values.get('healthcheck')
                 changed_fields.add("healthcheck")
diff --git a/rootfs/api/models/limit.py b/rootfs/api/models/limit.py
index 7727bf8cd..14119d5ac 100644
--- a/rootfs/api/models/limit.py
+++ b/rootfs/api/models/limit.py
@@ -115,6 +115,7 @@ class LimitPlan(AuditedModel):
     requests = models.JSONField(default=dict)
     annotations = models.JSONField(default=dict)
     node_selector = models.JSONField(default=dict)
+    runtime_class_name = models.CharField(max_length=63, default="")
     pod_security_context = models.JSONField(default=dict)
     container_security_context = models.JSONField(default=dict)
 
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index d85fcae97..a7d12d48a 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -67,18 +67,16 @@ def version_name(self):
     def get_runners(self, ptypes):
         results = []
         ptypes = self.ptypes if not ptypes else ptypes
-        for run in self.build.dryccfile.get('run', []):
-            for ptype in ptypes:
-                when_ptypes = run.get('when', {}).get('ptypes', [])
-                if not when_ptypes or ptype in when_ptypes:
-                    image = run.get('image', self.build.get_image(ptype))
-                    results.append({
-                        'image': self.build.get_image(image, default_image=image),
-                        'args': run.get('args', []),
-                        'command': run.get('command', []),
-                        'timeout': run.get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
-                    })
-                    break
+        for ptype, run in self.build.dryccfile.get('run', {}).items():
+            if ptype in ptypes:
+                image = run.get('image', self.build.get_image(ptype))
+                results.append({
+                    'ptype': ptype,
+                    'image': self.build.get_image(image, default_image=image),
+                    'args': run.get('args', []),
+                    'command': run.get('command', []),
+                    'timeout': run.get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
+                })
         return results
 
     def add_condition(self, **kwargs):
diff --git a/rootfs/api/serializers/schemas/dryccfile.py b/rootfs/api/serializers/schemas/dryccfile.py
index c1554f4e3..83d2cdaaa 100644
--- a/rootfs/api/serializers/schemas/dryccfile.py
+++ b/rootfs/api/serializers/schemas/dryccfile.py
@@ -1,5 +1,4 @@
 PROCTYPE_REGEX = r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)$'
-
 SCHEMA = {
     "$schema": "http://json-schema.org/schema#",
     "type": "object",
@@ -14,25 +13,71 @@
                     },
                     "additionalProperties": False,
                 },
-                "config": {"type": "object"}
+                "config": {
+                    "oneOf": [
+                        {
+                            "type": "object",
+                            "patternProperties": {
+                                PROCTYPE_REGEX: {
+                                    "type": "array",
+                                    "items": {
+                                        "type": "object",
+                                        "properties": {
+                                            "name": {"type": "string"},
+                                            "value": {"type": "string"},
+                                        },
+                                        "additionalProperties": False,
+                                    },
+                                },
+                            },
+                            "minProperties": 1,
+                            "additionalProperties": False,
+                        },
+                        {
+                            "type": "array",
+                            "items": {
+                                "type": "object",
+                                "properties": {
+                                    "name": {"type": "string"},
+                                    "value": {"type": "string"},
+                                },
+                                "additionalProperties": False,
+                            },
+                        },
+                    ]
+                }
             },
         },
-        "run": {
+        "config": {
             "type": "array",
             "items": {
                 "type": "object",
                 "properties": {
-                    "image": {"type": "string"},
-                    "command": {
-                        "type": "array",
-                        "items": {"type": "string"},
-                    },
-                    "args": {
-                        "type": "array",
-                        "items": {"type": "string"},
+                    "name": {"type": "string"},
+                    "group": {"type": "string"},
+                    "value": {"type": "string"},
+                },
+                "additionalProperties": False,
+            },
+        },
+        "run": {
+            "type": "object",
+            "patternProperties": {
+                PROCTYPE_REGEX: {
+                    "properties": {
+                        "image": {"type": "string"},
+                        "command": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                        },
+                        "args": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                        }
                     }
                 },
             },
+            "additionalProperties": False,
         },
         "deploy": {
             "type": "object",
@@ -48,6 +93,23 @@
                             "type": "array",
                             "items": {"type": "string"},
                         },
+                        "config": {
+                            "env": {
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "name": {"type": "string"},
+                                        "value": {"type": "string"},
+                                    },
+                                    "additionalProperties": False,
+                                },
+                            },
+                            "ref": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                            }
+                        },
                     }
                 },
             },
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index d8ce0c0e8..160a8d206 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -332,8 +332,6 @@
 
 DRYCC_APP_STORAGE_CLASS = os.environ.get('DRYCC_APP_STORAGE_CLASS', "")
 
-DRYCC_APP_RUNTIME_CLASS = os.environ.get('DRYCC_APP_RUNTIME_CLASS', "")
-
 DRYCC_APP_DNS_POLICY = os.environ.get('DRYCC_APP_DNS_POLICY', "")
 
 DRYCC_APP_POD_EXEC_TIMEOUT = int(os.environ.get('DRYCC_APP_POD_EXEC_TIMEOUT', "3600"))
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 515fdb488..d37ca0281 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -750,14 +750,15 @@ def test_dryccfile_ok(self, mock_requests):
             'dryccfile': {
                 "build": {
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                    "config": [
+                        {"name": "FOO", "value": "bar"},
+                        {"name": "RAILS_ENV", "value": "development"},
+                    ],
+                },
+                "run": {
+                    "web": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                    "worker": {"command": ["./deployment-tasks.sh"], "image": run_image},
                 },
-                "run": [
-                    {
-                        "command": ["./deployment-tasks.sh"],
-                        "image": run_image,
-                    }
-                ],
                 "deploy": {
                     "web": {
                         "command": ["bash", "-c"],
@@ -803,7 +804,10 @@ def test_dryccfile_format(self, mock_requests):
             'dryccfile': {
                 "build": {
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                    "config": [
+                        {"name": "FOO", "value": "bar"},
+                        {"name": "RAILS_ENV", "value": "development"},
+                    ],
                 },
             }
         }
@@ -829,12 +833,10 @@ def test_dryccfile_format(self, mock_requests):
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
 
-            body['dryccfile']['run'] = [
-                {
-                    'command': ["bash", "-c"],
-                    'args': ["ls /"]
-                }
-            ]
+            body['dryccfile']['run'] = {
+                "web": {'command': ["bash", "-c"], 'args': ["ls /"]},
+                "web-new": {'command': ["bash", "-c"], 'args': ["ls /"]},
+            }
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
             body['dryccfile']['deploy'] = {}
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index fe196448f..930d061d1 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -825,7 +825,10 @@ def test_config_from_dryccfile(self, mock_requests):
             'dryccfile': {
                 "build": {
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                    "config": [
+                        {"name": "FOO", "value": "bar"},
+                        {"name": "RAILS_ENV", "value": "development"},
+                    ],
                 },
                 'deploy': {
                     'web': {
@@ -848,16 +851,12 @@ def test_config_from_dryccfile(self, mock_requests):
         self.assertEqual(release.failed, False)
         self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
         # set env by dryccfile
-        build_body['dryccfile']['config'] = {
-            "mygroup1": [
-                {"name": "GROUP", "value": "g1"},
-                {"name": "DEBUG", "value": "tr"},
-            ],
-            "mygroup2": [
-                {"name": "TEST1", "value": "g1"},
-                {"name": "TEST2", "value": "tr"},
-            ],
-        }
+        build_body['dryccfile']['config'] = [
+            {"name": "GROUP", "group": "mygroup1", "value": "g1"},
+            {"name": "DEBUG", "group": "mygroup1", "value": "tr"},
+            {"name": "TEST1", "group": "mygroup2", "value": "g1"},
+            {"name": "TEST2", "group": "mygroup2", "value": "tr"},
+        ]
         build_body['dryccfile']['deploy']['web']['config'] = {
             'env': [
                 {'name': "PENV1", 'value': 'web'},
@@ -919,7 +918,7 @@ def test_config_from_dryccfile(self, mock_requests):
         self.assertEqual(release.config.values_refs, release.previous().config.values_refs)
         # set empty
         new_build_body = copy.deepcopy(build_body)
-        new_build_body['dryccfile']['config'] = {}
+        new_build_body['dryccfile']['config'] = []
         new_build_body['dryccfile']['deploy']['web']['config'] = {}
         new_build_body['dryccfile']['deploy']['web']['healthcheck'] = {}
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index f63a08255..42d774ef8 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -144,14 +144,19 @@ def test_get_image(self, mock_requests):
             'dryccfile': {
                 "build": {
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                    "config": [
+                        {"name": "FOO", "value": "bar"},
+                        {"name": "RAILS_ENV", "value": "development"},
+                    ],
+                },
+                "run": {
+                    "web": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                    "worker": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                    "worker-1": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                    "worker-2": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                    "worker-3": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                    "worker-4": {"command": ["./deployment-tasks.sh"], "image": "worker"},
                 },
-                "run": [
-                    {
-                        "command": ["./deployment-tasks.sh"],
-                        "image": "worker",
-                    }
-                ],
                 "deploy": {
                     "web": {
                         "command": ["bash", "-c"],
@@ -702,19 +707,17 @@ def test_run(self, mock_requests):
         app = App.objects.get(id=app_id)
         user = User.objects.get(username='autotest')
         dryccfile = {
-            "run": [
-                {
+            "run": {
+                "web": {
                     "args": ["sleep", "60s"],
                     "image": "registry.drycc.cc/drycc/base:bookworm-web",
-                    "when": {"ptypes": ["web"]}
                 },
-                {
+                "task": {
                     "args": ["sleep", "60s"],
                     "image": "registry.drycc.cc/drycc/base:bookworm-task",
-                    "when": {"ptypes": ["task"]},
                     "timeout": 3000
                 }
-            ],
+            },
             "deploy": {
                 "web": {
                     "image": "registry.drycc.cc/drycc/python-dev",
@@ -742,7 +745,7 @@ def test_run(self, mock_requests):
         self.assertEqual(runners[0]["timeout"], 3000)
 
         # change timeout
-        dryccfile['run'][0]['timeout'] = 3600
+        dryccfile['run']['web']['timeout'] = 3600
         dryccfile['deploy']['web']['image'] = 'registry.drycc.cc/drycc/python-dev:latest'
         build = Build.objects.create(
             owner=user, app=app, image="qwerty", procfile={},
@@ -768,14 +771,15 @@ def test_release_diff_ptypes(self, mock_requests):
             'dryccfile': {
                 "build": {
                     "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": {"RAILS_ENV": "development", "FOO": "bar"}
+                     "config": [
+                        {"name": "FOO", "value": "bar"},
+                        {"name": "RAILS_ENV", "value": "development"},
+                    ],
+                },
+                "run": {
+                    "web": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                    "worker": {"command": ["./deployment-tasks.sh"], "image": run_image},
                 },
-                "run": [
-                    {
-                        "command": ["./deployment-tasks.sh"],
-                        "image": run_image,
-                    }
-                ],
                 "deploy": {
                     "web": {
                         "command": ["bash", "-c"],
@@ -799,14 +803,13 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, ["web", "worker"])
 
-        body['dryccfile']['config'] = {
-            'g1': [
-                {
-                    'name': 'G1',
-                    'value': 'g1',
-                },
-            ]
-        }
+        body['dryccfile']['config'] = [
+            {
+                'name': 'G1',
+                'group': 'g1',
+                'value': 'g1',
+            },
+        ]
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -819,14 +822,13 @@ def test_release_diff_ptypes(self, mock_requests):
         body['dryccfile']['deploy']['worker']['config'] = {
             'ref': ['g1']
         }
-        body['dryccfile']['config'] = {
-            'g1': [
-                {
-                    'name': 'G1',
-                    'value': 'g1',
-                },
-            ]
-        }
+        body['dryccfile']['config'] = [
+            {
+                'name': 'G1',
+                'group': 'g1',
+                'value': 'g1',
+            },
+        ]
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -836,14 +838,13 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, ["worker"])
         # env cover, only change web
-        body['dryccfile']['config'] = {
-            'global': [
-                {
-                    'name': 'G1',
-                    'value': 'g1',
-                },
-            ]
-        }
+        body['dryccfile']['config'] = [
+            {
+                'name': 'G1',
+                'group': 'global',
+                'value': 'g1',
+            },
+        ]
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -853,14 +854,13 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, ["web"])
         # add a global env
-        body['dryccfile']['config'] = {
-            'global': [
-                {
-                    'name': 'G2',
-                    'value': 'g2',
-                },
-            ]
-        }
+        body['dryccfile']['config'] = [
+            {
+                'name': 'G2',
+                'group': 'global',
+                'value': 'g2',
+            },
+        ]
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)

From 0998f3f7514e4f24ee96ed1e8a63cd7de50984aa Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 2 Jan 2025 11:42:25 +0800
Subject: [PATCH 110/200] feat(dryccfile): dryccfile support pipeline

---
 rootfs/api/models/build.py                  |  63 ++---
 rootfs/api/models/release.py                |  24 +-
 rootfs/api/serializers/__init__.py          |  15 +-
 rootfs/api/serializers/schemas/dryccfile.py | 158 +++++-------
 rootfs/api/tests/test_build.py              | 118 +++++----
 rootfs/api/tests/test_config.py             | 101 ++++----
 rootfs/api/tests/test_procfile.py           |  60 +++--
 rootfs/api/tests/test_release.py            | 255 +++++++++++---------
 8 files changed, 412 insertions(+), 382 deletions(-)

diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 810ed90ea..5b2060ee8 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -45,7 +45,7 @@ def type(self):
     @property
     def ptypes(self):
         if self.dryccfile:
-            return list(self.dryccfile['deploy'].keys())
+            return list([pipeline['ptype'] for pipeline in self.dryccfile['pipeline'].values()])
         if self.procfile:
             return list(self.procfile.keys())
         return [PTYPE_WEB]
@@ -64,14 +64,24 @@ def version(self):
         return 'git-{}'.format(self.sha) if self.source_based else 'latest'
 
     def get_image(self, ptype, default_image=None):
-        docker = self.dryccfile.get('build', {}).get('docker', {})
-        if ptype in docker:
-            if ptype == 'web':
-                return self.image
-            else:
-                return f'{self.image}-{ptype}'
+        pipeline = self.get_pipeline(ptype)
+        if pipeline and 'build' in pipeline:
+            return self.image if ptype == 'web' else f'{self.image}-{ptype}'
         return default_image if default_image else self.image
 
+    def get_pipeline(self, ptype):
+        pipelines = self.get_pipelines([ptype])
+        if len(pipelines) > 0:
+            return pipelines[0]
+        return {}
+
+    def get_pipelines(self, ptypes):
+        results = []
+        for pipeline in self.dryccfile.get('pipeline', {}).values():
+            if pipeline['ptype'] in ptypes:
+                results.append(pipeline)
+        return results
+
     def create_release(self, user, *args, **kwargs):
         latest_release = Release.latest(self.app)
         try:
@@ -110,38 +120,31 @@ def _get_or_create_config(self):
         """
         dryccfile to config
         """
-        config_values, config_values_ref, config_healthcheck, changed_fields = (
-            self.dryccfile.get('config', []), {}, {}, set())
-        if 'config' in self.dryccfile:
+        config_values, config_values_ref, changed_fields = [], {}, set()
+        for group, envs in self.dryccfile.get('config', {}).items():
+            for key, value in envs.items():
+                config_values.append({"name": key, "group": group, "value": value})
             changed_fields.update(["values", "values_refs"])
-        replace_values_ptypes, replace_healthcheck_ptypes = set(), set()
-        for ptype, values in self.dryccfile.get('deploy', {}).items():
-            if 'config' in values:
-                for value in values.get('config', {}).get('env', []):
-                    value['ptype'] = ptype
-                    config_values.append(value)
-                    replace_values_ptypes.add(ptype)
-                for config_ref in values.get('config', {}).get('ref', []):
-                    if ptype not in config_values_ref:
-                        config_values_ref[ptype] = [config_ref]
+
+        replace_values_ptypes = set()
+        for pipeline in self.dryccfile.get('pipeline', {}).values():
+            if 'env' in pipeline or 'config' in pipeline:
+                for key, value in pipeline.get('env', {}).items():
+                    config_values.append({"name": key, "ptype": pipeline['ptype'], "value": value})
+                for config_ref in pipeline.get('config', []):
+                    if pipeline['ptype'] not in config_values_ref:
+                        config_values_ref[pipeline['ptype']] = [config_ref]
                     else:
-                        config_values_ref[ptype].append(config_ref)
-                replace_values_ptypes.add(ptype)
+                        config_values_ref[pipeline['ptype']].append(config_ref)
+                replace_values_ptypes.add(pipeline['ptype'])
                 changed_fields.update(["values", "values_refs"])
-            if 'healthcheck' in values:
-                config_healthcheck[ptype] = values.get('healthcheck')
-                changed_fields.add("healthcheck")
-                replace_healthcheck_ptypes.add(ptype)
 
         old_config = self.app.release_set.filter(failed=False).latest().config
         if not changed_fields:
             return old_config
         config = Config(
-            owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref,
-            healthcheck=config_healthcheck,
-        )
+            owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref)
         config.merge_field("values", old_config, replace_values_ptypes)
         config.merge_field("values_refs", old_config, replace_values_ptypes)
-        config.merge_field("healthcheck", old_config, replace_healthcheck_ptypes)
         config.save(ignore_update_fields=changed_fields)
         return config
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index a7d12d48a..a1c3ab47f 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -67,15 +67,15 @@ def version_name(self):
     def get_runners(self, ptypes):
         results = []
         ptypes = self.ptypes if not ptypes else ptypes
-        for ptype, run in self.build.dryccfile.get('run', {}).items():
-            if ptype in ptypes:
-                image = run.get('image', self.build.get_image(ptype))
+        for pipeline in self.build.get_pipelines(ptypes):
+            if 'run' in pipeline:
+                image = pipeline['run'].get('image', self.build.get_image(pipeline['ptype']))
                 results.append({
-                    'ptype': ptype,
+                    'ptype': pipeline['ptype'],
                     'image': self.build.get_image(image, default_image=image),
-                    'args': run.get('args', []),
-                    'command': run.get('command', []),
-                    'timeout': run.get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
+                    'args': pipeline['run'].get('args', []),
+                    'command': pipeline['run'].get('command', []),
+                    'timeout': pipeline['run'].get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
                 })
         return results
 
@@ -93,21 +93,19 @@ def add_condition(self, **kwargs):
 
     def get_deploy_image(self, ptype):
         """
-        In the deploy phase of dryccfile
         Return the kubernetes "container image" to be sent off to the scheduler.
         """
-        image = self.build.dryccfile.get('deploy', {}).get(ptype, {}).get(
+        image = self.build.get_pipeline(ptype).get('deploy', {}).get(
             'image', self.build.get_image(ptype))
         return self.build.get_image(image, default_image=image)
 
     def get_deploy_args(self, ptype):
         """
-        In the deploy phase of dryccfile
         Return the kubernetes "container arguments" to be sent off to the scheduler.
         """
         if self.build is not None:
             if self.build.dryccfile:
-                return self.build.dryccfile['deploy'].get(ptype, {}).get('args', [])
+                return self.build.get_pipeline(ptype).get('deploy', {}).get('args', [])
             else:
                 # dockerfile or container image
                 if self.build.dockerfile or not self.build.sha:
@@ -119,11 +117,9 @@ def get_deploy_args(self, ptype):
 
     def get_deploy_command(self, ptype):
         """
-        In the deploy phase of dryccfile
         Return the kubernetes "container command" to be sent off to the scheduler.
         """
-        return self.build.dryccfile.get(
-            'deploy', {}).get(ptype, {}).get('command', [])
+        return self.build.get_pipeline(ptype).get('deploy', {}).get('command', [])
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 60964aff8..e1e3c6b6b 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -24,7 +24,7 @@
 from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
 from .schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
 from .schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
-from .schemas.dryccfile import (SCHEMA as DRYCCFILE_SCHEMA, PROCTYPE_REGEX)
+from .schemas.dryccfile import (SCHEMA as DRYCCFILE_SCHEMA, PROCTYPE_REGEX, CONFIGKEY_REGEX)
 
 
 User = get_user_model()
@@ -42,7 +42,7 @@
 PROCTYPE_MISMATCH_MSG = "Process types can only supports: %s" % PROCTYPE_MATCH.pattern
 
 TAGVAL_MATCH = re.compile(r'^(?:[a-zA-Z\d][-\.\w]{0,61})?[a-zA-Z\d]$')
-CONFIGKEY_MATCH = re.compile(r'^[A-z0-9_\-\.]+$', re.IGNORECASE)
+CONFIGKEY_MATCH = re.compile(CONFIGKEY_REGEX, re.IGNORECASE)
 CONFIGKEY_MISMATCH_MSG = ("Config key can only supports: %s" % CONFIGKEY_MATCH.pattern)
 CONFIG_LIMITS_MISMATCH_MSG = "The limit plan {} does not exist"
 
@@ -231,12 +231,15 @@ def validate_procfile(data):
 
     @staticmethod
     def validate_dryccfile(data):
+        ptypes = set()
         if data:
             validate_json(data, DRYCCFILE_SCHEMA, serializers.ValidationError)
-            ptypes = set().union(data.get("deploy", {}).keys())
-            ptypes = ptypes.union(data.get("build", {}).get("docker", {}).keys())
-            for ptype in ptypes:
-                validate_ptype(ptype)
+            for pipeline in data['pipeline'].values():
+                if pipeline['ptype'] in ptypes:
+                    raise serializers.ValidationError("Pipeline's ptype must be unique")
+                else:
+                    validate_ptype(pipeline['ptype'])
+                    ptypes.add(pipeline['ptype'])
         return data
 
 
diff --git a/rootfs/api/serializers/schemas/dryccfile.py b/rootfs/api/serializers/schemas/dryccfile.py
index 83d2cdaaa..bb9729d2b 100644
--- a/rootfs/api/serializers/schemas/dryccfile.py
+++ b/rootfs/api/serializers/schemas/dryccfile.py
@@ -1,121 +1,85 @@
+
+FILENAME_REGEX = r'^(?P<name>[a-z0-9]+(\.[a-z0-9]+)*)$'
 PROCTYPE_REGEX = r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)$'
+CONFIGKEY_REGEX = r'^[A-z0-9_\-\.]+$'
+CONFIGENV_SCHEMA = {
+    "type": "object",
+    "patternProperties": {
+        CONFIGKEY_REGEX: {"type": "string"},
+    },
+}
 SCHEMA = {
     "$schema": "http://json-schema.org/schema#",
     "type": "object",
     "properties": {
-        "build": {
+        "config": {
+            "type": "object",
+            "patternProperties": {
+                FILENAME_REGEX: CONFIGENV_SCHEMA,
+            },
+        },
+        "pipeline": {
             "type": "object",
-            "properties": {
-                "docker": {
+            "patternProperties": {
+                FILENAME_REGEX: {
                     "type": "object",
-                    "patternProperties": {
-                        PROCTYPE_REGEX: {"type": "string"},
-                    },
-                    "additionalProperties": False,
-                },
-                "config": {
-                    "oneOf": [
-                        {
+                    "properties": {
+                        "kind": {"type": "string", "pattern": "^pipeline$"},
+                        "ptype": {"type": "string", "pattern": PROCTYPE_REGEX},
+                        "build": {
                             "type": "object",
-                            "patternProperties": {
-                                PROCTYPE_REGEX: {
+                            "properties": {
+                                "docker": {"type": "string"},
+                                "arg": CONFIGENV_SCHEMA,
+                            },
+                            "additionalProperties": False,
+                        },
+                        "env": CONFIGENV_SCHEMA,
+                        "run": {
+                            "type": "object",
+                            "properties": {
+                                "command": {
                                     "type": "array",
-                                    "items": {
-                                        "type": "object",
-                                        "properties": {
-                                            "name": {"type": "string"},
-                                            "value": {"type": "string"},
-                                        },
-                                        "additionalProperties": False,
-                                    },
+                                    "items": {"type": "string"},
                                 },
+                                "args": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
+                                },
+                                "image": {"type": "string"},
+                                "timeout": {"type": "integer"},
                             },
-                            "minProperties": 1,
                             "additionalProperties": False,
                         },
-                        {
+                        "config": {
                             "type": "array",
                             "items": {
-                                "type": "object",
-                                "properties": {
-                                    "name": {"type": "string"},
-                                    "value": {"type": "string"},
-                                },
-                                "additionalProperties": False,
+                                "type": "string",
+                                "pattern": FILENAME_REGEX,
                             },
                         },
-                    ]
-                }
-            },
-        },
-        "config": {
-            "type": "array",
-            "items": {
-                "type": "object",
-                "properties": {
-                    "name": {"type": "string"},
-                    "group": {"type": "string"},
-                    "value": {"type": "string"},
-                },
-                "additionalProperties": False,
-            },
-        },
-        "run": {
-            "type": "object",
-            "patternProperties": {
-                PROCTYPE_REGEX: {
-                    "properties": {
-                        "image": {"type": "string"},
-                        "command": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                        },
-                        "args": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                        }
-                    }
-                },
-            },
-            "additionalProperties": False,
-        },
-        "deploy": {
-            "type": "object",
-            "patternProperties": {
-                PROCTYPE_REGEX: {
-                    "properties": {
-                        "image": {"type": "string"},
-                        "command": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                        },
-                        "args": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                        },
-                        "config": {
-                            "env": {
-                                "type": "array",
-                                "items": {
-                                    "type": "object",
-                                    "properties": {
-                                        "name": {"type": "string"},
-                                        "value": {"type": "string"},
-                                    },
-                                    "additionalProperties": False,
+                        "deploy": {
+                            "type": "object",
+                            "properties": {
+                                "command": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
                                 },
+                                "args": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
+                                },
+                                "image": {"type": "string"},
                             },
-                            "ref": {
-                                "type": "array",
-                                "items": {"type": "string"},
-                            }
-                        },
-                    }
+                            "additionalProperties": False,
+                        }
+                    },
+                    "additionalProperties": False,
+                    "required": ["kind", "ptype", "deploy"],
                 },
             },
-            "minProperties": 1,
-            "additionalProperties": False,
         },
     },
-    "required": ["deploy"],
+    "required": ["pipeline"],
+    "additionalProperties": False,
 }
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index d37ca0281..fbd7c8625 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -747,31 +747,44 @@ def test_dryccfile_ok(self, mock_requests):
             'image': 'autotest/example',
             'stack': 'heroku-18',
             'sha': 'a'*40,
-            'dryccfile': {
-                "build": {
-                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": [
-                        {"name": "FOO", "value": "bar"},
-                        {"name": "RAILS_ENV", "value": "development"},
-                    ],
-                },
-                "run": {
-                    "web": {"command": ["./deployment-tasks.sh"], "image": run_image},
-                    "worker": {"command": ["./deployment-tasks.sh"], "image": run_image},
-                },
-                "deploy": {
-                    "web": {
-                        "command": ["bash", "-c"],
-                        "args": ["bundle exec puma -C config/puma.rb"],
-                        "image": "127.0.0.1:7070/myapp/web:git-123fsa1"
+            "dryccfile": {
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": "127.0.0.1:7070/myapp/web:git-123fsa1",
+                        },
                     },
-                    "worker": {
-                        "command": ["bash", "-c"],
-                        "args": ["python myworker.py"],
-                        "image": "127.0.0.1:7070/myapp/worker:git-123fsa1"
-                    }
-                }
-            }
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["python myworker.py"],
+                            "image": "127.0.0.1:7070/myapp/worker:git-123fsa1"
+                        },
+                    },
+                },
+            },
         }
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
@@ -802,14 +815,31 @@ def test_dryccfile_format(self, mock_requests):
             'stack': 'heroku-18',
             'sha': 'a'*40,
             'dryccfile': {
-                "build": {
-                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": [
-                        {"name": "FOO", "value": "bar"},
-                        {"name": "RAILS_ENV", "value": "development"},
-                    ],
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                    },
                 },
-            }
+            },
         }
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
@@ -817,29 +847,19 @@ def test_dryccfile_format(self, mock_requests):
             url = f"/v2/apps/{app_id}/build"
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
-            body['dryccfile']['deploy'] = {
-                "web-new": {
-                    'image': "127.0.0.1/cat/cat"
-                }
-            }
+            body['dryccfile']['pipeline']["web.yaml"]['deploy'] = {'image': "127.0.0.1/cat/cat"}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 201, response.data)
-            del body['dryccfile']['deploy']['web-new']
-            body['dryccfile']['deploy'] = {
-                "web": {
-                    'image': "127.0.0.1/cat/cat"
-                }
-            }
+            self.assertEqual(response.status_code, 400, response.data)
+            body['dryccfile']['pipeline']["worker.yaml"]['deploy'] = {'image': "127.0.0.1/cat/cat"}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
-
-            body['dryccfile']['run'] = {
-                "web": {'command': ["bash", "-c"], 'args': ["ls /"]},
-                "web-new": {'command': ["bash", "-c"], 'args': ["ls /"]},
-            }
+            body['dryccfile']['pipeline']["web.yaml"]['run'] = {
+                'command': ["bash", "-c"], 'args': ["ls /"]}
+            body['dryccfile']['pipeline']["worker.yaml"]['run'] = {
+                'command': ["bash", "-c"], 'args': ["ls /"]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
-            body['dryccfile']['deploy'] = {}
+            body['dryccfile']['pipeline']["web.yaml"] = {}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
             body['dryccfile'] = {}
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 930d061d1..6fe1308a8 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -823,18 +823,36 @@ def test_config_from_dryccfile(self, mock_requests):
             'stack': 'heroku-18',
             'sha': 'a'*40,
             'dryccfile': {
-                "build": {
-                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": [
-                        {"name": "FOO", "value": "bar"},
-                        {"name": "RAILS_ENV", "value": "development"},
-                    ],
-                },
-                'deploy': {
-                    'web': {
-                        'image': "127.0.0.1/cat/cat"
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "deploy": {
+                            'image': "127.0.0.1/cat/cat"
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "deploy": {
+                            'image': "127.0.0.1/cat/cat"
+                        },
                     },
-                }
+                },
             },
         }
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
@@ -851,19 +869,23 @@ def test_config_from_dryccfile(self, mock_requests):
         self.assertEqual(release.failed, False)
         self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
         # set env by dryccfile
-        build_body['dryccfile']['config'] = [
-            {"name": "GROUP", "group": "mygroup1", "value": "g1"},
-            {"name": "DEBUG", "group": "mygroup1", "value": "tr"},
-            {"name": "TEST1", "group": "mygroup2", "value": "g1"},
-            {"name": "TEST2", "group": "mygroup2", "value": "tr"},
-        ]
-        build_body['dryccfile']['deploy']['web']['config'] = {
-            'env': [
-                {'name': "PENV1", 'value': 'web'},
-                {'name': "PENV2", 'value': 'web'},
-            ],
-            'ref': ['mygroup1', 'mygroup2']
+        build_body['dryccfile']['config'] = {
+            "mygroup1": {
+                "GROUP": "g1",
+                "DEBUG": "tr",
+            },
+            "mygroup2": {
+                "TEST1": "g1",
+                "TEST2": "tr",
+            },
+        }
+        build_body['dryccfile']['pipeline']['web.yaml']['env'] = {
+            "PENV1": "web",
+            "PENV2": "web",
         }
+        build_body['dryccfile']['pipeline']['web.yaml']['config'] = [
+            "mygroup1", "mygroup2"
+        ]
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             url = f"/v2/apps/{app_id}/build"
@@ -879,33 +901,12 @@ def test_config_from_dryccfile(self, mock_requests):
                 'PENV2': 'web', "WEBSITE": "www.drycc.cc",
             }
         )
-        build_body['dryccfile']['deploy']['web']['healthcheck'] = {
-            'livenessProbe': {
-                'httpGet': {
-                    'path': '/healthz',
-                    'port': 8080,
-                },
-                'initialDelaySeconds': 3,
-                'periodSeconds': 3,
-            }
-        }
-        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
-            mock_kube.return_value = ['up', 'down']
-            url = f"/v2/apps/{app_id}/build"
-            response = self.client.post(url, build_body)
-            self.assertEqual(response.status_code, 201, response.data)
 
-        release = app.release_set.latest()
-        self.assertEqual(release.failed, False)
-        self.assertEqual(
-            release.config.healthcheck['web'],
-            build_body['dryccfile']['deploy']['web']['healthcheck'],
-        )
         # test use old config and healthcheck
         new_build_body = copy.deepcopy(build_body)
         del new_build_body['dryccfile']['config']
-        del new_build_body['dryccfile']['deploy']['web']['config']
-        del new_build_body['dryccfile']['deploy']['web']['healthcheck']
+        del new_build_body['dryccfile']['pipeline']['web.yaml']['env']
+        del new_build_body['dryccfile']['pipeline']['web.yaml']['config']
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             url = f"/v2/apps/{app_id}/build"
@@ -913,14 +914,13 @@ def test_config_from_dryccfile(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.failed, False)
-        self.assertEqual(release.config.healthcheck, release.previous().config.healthcheck)
         self.assertEqual(release.config.values, release.previous().config.values)
         self.assertEqual(release.config.values_refs, release.previous().config.values_refs)
         # set empty
         new_build_body = copy.deepcopy(build_body)
-        new_build_body['dryccfile']['config'] = []
-        new_build_body['dryccfile']['deploy']['web']['config'] = {}
-        new_build_body['dryccfile']['deploy']['web']['healthcheck'] = {}
+        new_build_body['dryccfile']['config'] = {}
+        new_build_body['dryccfile']['pipeline']['web.yaml']['env'] = {}
+        new_build_body['dryccfile']['pipeline']['web.yaml']['config'] = []
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             url = f"/v2/apps/{app_id}/build"
@@ -928,6 +928,5 @@ def test_config_from_dryccfile(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.failed, False)
-        self.assertEqual(release.config.healthcheck, {'web': {}})
         self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
         self.assertEqual(release.config.values_refs, {})
diff --git a/rootfs/api/tests/test_procfile.py b/rootfs/api/tests/test_procfile.py
index 2b2d01423..cf4f3eb8b 100644
--- a/rootfs/api/tests/test_procfile.py
+++ b/rootfs/api/tests/test_procfile.py
@@ -30,33 +30,57 @@ def test_ptype_ok(self):
 
     def test_dryccfile_ptype(self):
         dryccfile_1 = {
-            "build": {
-                "docker": {
-                    "web": "Dockerfile",
-                    "task": "Dockerfile.task"
-                },
-            },
-            "deploy": {
-                "web": {
-                    "args": ["python", "-m", "http.server", "5000"]
-                },
-                "task": {
-                    "command": ["sleep"],
-                    "args": ["infinity"]
+            "pipeline": {
+                "web.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "web",
+                    "build": {
+                        "docker": "Dockerfile",
+                    },
+                    "deploy": {
+                        "args": ["python", "-m", "http.server", "5000"],
+                    },
                 },
+                "worker.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "worker",
+                    "build": {
+                        "docker": "Dockerfile.task",
+                    },
+                    "deploy": {
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    },
+                }
             }
         }
         validate_dryccfile = BuildSerializer().validate_dryccfile
         self.assertEqual(validate_dryccfile(dryccfile_1), dryccfile_1)
-        dryccfile_1["build"]["docker"]["w"] = "Dockerfile.w"
+        dryccfile_1["pipeline"]["w.yaml"] = {
+            "kind": "pipeline",
+            "ptype": "w",
+            "build": {"docker": "Dockerfile.task"},
+        }
         self.assertException(
                 validate_dryccfile, serializers.ValidationError, dryccfile_1)
-        del dryccfile_1["build"]["docker"]["w"]
-        dryccfile_1["deploy"]["w"] = {"args": ["python", "-m", "http.server", "5000"]}
+        del dryccfile_1["pipeline"]["w.yaml"]
+
+        dryccfile_1["pipeline"]["w.yaml"] = {
+            "kind": "pipeline",
+            "ptype": "w",
+            "build": {"docker": "Dockerfile.task"},
+            "deploy": {"args": ["python", "-m", "http.server", "5000"]},
+        }
         self.assertException(
                 validate_dryccfile, serializers.ValidationError, dryccfile_1)
-        del dryccfile_1["deploy"]["w"]
-        dryccfile_1["deploy"]["w-new-"] = {"args": ["python", "-m", "http.server", "5000"]}
+        dryccfile_1["pipeline"]["w.yaml"]
+
+        dryccfile_1["pipeline"]["wnew.yaml"] = {
+            "kind": "pipeline",
+            "ptype": "w-new-",
+            "build": {"docker": "Dockerfile.task"},
+            "deploy": {"args": ["python", "-m", "http.server", "5000"]},
+        }
         self.assertException(
                 validate_dryccfile, serializers.ValidationError, dryccfile_1)
 
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 42d774ef8..825b18918 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -142,46 +142,75 @@ def test_get_image(self, mock_requests):
             'stack': 'heroku-18',
             'sha': 'a'*40,
             'dryccfile': {
-                "build": {
-                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                    "config": [
-                        {"name": "FOO", "value": "bar"},
-                        {"name": "RAILS_ENV", "value": "development"},
-                    ],
-                },
-                "run": {
-                    "web": {"command": ["./deployment-tasks.sh"], "image": "worker"},
-                    "worker": {"command": ["./deployment-tasks.sh"], "image": "worker"},
-                    "worker-1": {"command": ["./deployment-tasks.sh"], "image": "worker"},
-                    "worker-2": {"command": ["./deployment-tasks.sh"], "image": "worker"},
-                    "worker-3": {"command": ["./deployment-tasks.sh"], "image": "worker"},
-                    "worker-4": {"command": ["./deployment-tasks.sh"], "image": "worker"},
-                },
-                "deploy": {
-                    "web": {
-                        "command": ["bash", "-c"],
-                        "args": ["bundle exec puma -C config/puma.rb"],
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                        },
                     },
-                    "worker": {
-                        "command": ["bash", "-c"],
-                        "args": ["python myworker.py"],
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["python myworker.py"],
+                        },
                     },
-                    "worker-1": {
-                        "image": "worker"
+                    "worker-1.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-1",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "image": "worker",
+                        },
                     },
-                    "worker-2": {},
-                    "worker-3": {
-                        "command": ["bash", "-c"],
-                        "args": ["bundle exec puma -C config/puma.rb"],
-                        "image": "web"
+                    "worker-2.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-2",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {},
                     },
-                    "worker-4": {
-                        "command": ["bash", "-c"],
-                        "args": ["bundle exec puma -C config/puma.rb"],
-                        "image": "127.0.0.1:7070/myapp/web:git-123fsa1"
-                    }
-                }
-            }
+                    "worker-3.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-3",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": "web"
+                        },
+                    },
+                    "worker-4.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-4",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": "127.0.0.1:7070/myapp/web:git-123fsa1",
+                        },
+                    },
+                },
+            },
         }
         default_image = '127.0.0.1:5555/autotest/example:git-fadf1231'
         worker_image = "127.0.0.1:5555/autotest/example:git-fadf1231-worker"
@@ -707,28 +736,34 @@ def test_run(self, mock_requests):
         app = App.objects.get(id=app_id)
         user = User.objects.get(username='autotest')
         dryccfile = {
-            "run": {
-                "web": {
-                    "args": ["sleep", "60s"],
-                    "image": "registry.drycc.cc/drycc/base:bookworm-web",
+            "pipeline": {
+                "web.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "web",
+                    "run": {
+                        "args": ["sleep", "60s"],
+                        "image": "registry.drycc.cc/drycc/base:bookworm-web",
+                    },
+                    "deploy": {
+                        "image": "registry.drycc.cc/drycc/python-dev",
+                        "args": ["python", "-m", "http.server", "5000"]
+                    },
                 },
-                "task": {
-                    "args": ["sleep", "60s"],
-                    "image": "registry.drycc.cc/drycc/base:bookworm-task",
-                    "timeout": 3000
-                }
-            },
-            "deploy": {
-                "web": {
-                    "image": "registry.drycc.cc/drycc/python-dev",
-                    "args": ["python", "-m", "http.server", "5000"]
+                "task.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "task",
+                    "run": {
+                        "args": ["sleep", "60s"],
+                        "image": "registry.drycc.cc/drycc/base:bookworm-task",
+                        "timeout": 3000
+                    },
+                    "deploy": {
+                        "image": "docker.io/library/nginx:mainline-bookworm-perl",
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    },
                 },
-                "task": {
-                    "image": "docker.io/library/nginx:mainline-bookworm-perl",
-                    "command": ["sleep"],
-                    "args": ["infinity"]
-                }
-            }
+            },
         }
         build = Build.objects.create(
             owner=user, app=app, image="qwerty", procfile={},
@@ -745,8 +780,10 @@ def test_run(self, mock_requests):
         self.assertEqual(runners[0]["timeout"], 3000)
 
         # change timeout
-        dryccfile['run']['web']['timeout'] = 3600
-        dryccfile['deploy']['web']['image'] = 'registry.drycc.cc/drycc/python-dev:latest'
+        dryccfile['pipeline']['web.yaml']['run']['timeout'] = 3600
+        dryccfile['pipeline']['web.yaml']['deploy'] = {
+            'image': 'registry.drycc.cc/drycc/python-dev:latest',
+        }
         build = Build.objects.create(
             owner=user, app=app, image="qwerty", procfile={},
             sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
@@ -769,30 +806,43 @@ def test_release_diff_ptypes(self, mock_requests):
             'stack': 'heroku-18',
             'sha': 'a'*40,
             'dryccfile': {
-                "build": {
-                    "docker": {"web": "Dockerfile", "worker": "worker/Dockerfile"},
-                     "config": [
-                        {"name": "FOO", "value": "bar"},
-                        {"name": "RAILS_ENV", "value": "development"},
-                    ],
-                },
-                "run": {
-                    "web": {"command": ["./deployment-tasks.sh"], "image": run_image},
-                    "worker": {"command": ["./deployment-tasks.sh"], "image": run_image},
-                },
-                "deploy": {
-                    "web": {
-                        "command": ["bash", "-c"],
-                        "args": ["bundle exec puma -C config/puma.rb"],
-                        "image": web_image
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": web_image
+                        },
                     },
-                    "worker": {
-                        "command": ["bash", "-c"],
-                        "args": ["python myworker.py"],
-                        "image": worker_image
-                    }
-                }
-            }
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["python myworker.py"],
+                            "image": worker_image
+                        },
+                    },
+                },
+            },
         }
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
@@ -803,13 +853,7 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, ["web", "worker"])
 
-        body['dryccfile']['config'] = [
-            {
-                'name': 'G1',
-                'group': 'g1',
-                'value': 'g1',
-            },
-        ]
+        body['dryccfile']['config'] = {"g1": {"G1": "g1"}}
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -819,16 +863,9 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, [])
         # change worker env
-        body['dryccfile']['deploy']['worker']['config'] = {
-            'ref': ['g1']
-        }
-        body['dryccfile']['config'] = [
-            {
-                'name': 'G1',
-                'group': 'g1',
-                'value': 'g1',
-            },
-        ]
+        body['dryccfile']['pipeline']['worker.yaml']['config'] = ["g1"]
+        body['dryccfile']['config'] = {"g1": {"G1": "g1"}}
+
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -838,13 +875,7 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, ["worker"])
         # env cover, only change web
-        body['dryccfile']['config'] = [
-            {
-                'name': 'G1',
-                'group': 'global',
-                'value': 'g1',
-            },
-        ]
+        body['dryccfile']['config'] = {"global": {"G1": "g1"}}
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -854,13 +885,7 @@ def test_release_diff_ptypes(self, mock_requests):
         ptypes = release.diff_ptypes(release.ptypes)
         self.assertEqual(ptypes, ["web"])
         # add a global env
-        body['dryccfile']['config'] = [
-            {
-                'name': 'G2',
-                'group': 'global',
-                'value': 'g2',
-            },
-        ]
+        body['dryccfile']['config'] = {"global": {"G2": "g2"}}
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)
@@ -871,11 +896,7 @@ def test_release_diff_ptypes(self, mock_requests):
         self.assertEqual(set(ptypes), set(["web", "worker"]))
 
         # change web env
-        body['dryccfile']['deploy']['web']['config'] = {
-            'env': [
-                {'name': 'ENV1', 'value': 'env1'}
-            ]
-        }
+        body['dryccfile']['pipeline']['web.yaml']['env'] = {"ENV1": "env1"}
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
             response = self.client.post(url, body)

From 46452ee5a79a09072111676cf48347562f6fd765 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 6 Jan 2025 16:16:21 +0800
Subject: [PATCH 111/200] chore(deploy): add apply tasks

---
 rootfs/api/settings/production.py | 3 +++
 rootfs/api/utils.py               | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 160a8d206..6fecebee2 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -300,6 +300,9 @@
 
 IMAGE_PULL_POLICY = os.environ.get('IMAGE_PULL_POLICY', "IfNotPresent")
 
+# apply task size
+DRYCC_APPLY_TASKS = int(os.environ.get('DRYCC_APPLY_TASKS', '20'))
+
 # Drycc filer image
 # Provide get and put operations for `drycc volumes:client`
 DRYCC_FILER_IMAGE = os.environ.get('DRYCC_FILER_IMAGE', 'registry.drycc.cc/drycc/filer:canary')
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index 3d594a24b..444fcee8a 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -18,6 +18,7 @@
 import jsonschema
 from copy import deepcopy
 from django.db import models
+from django.conf import settings
 from django.core.cache import cache
 from asgiref.sync import sync_to_async
 from requests_toolbelt import user_agent
@@ -159,7 +160,7 @@ def apply_tasks(tasks):
     if not tasks:
         return
 
-    executor = concurrent.futures.ThreadPoolExecutor(5)
+    executor = concurrent.futures.ThreadPoolExecutor(settings.DRYCC_APPLY_TASKS)
     for future, callback in [(executor.submit(task[0]), task[1]) for task in tasks]:
         future.add_done_callback(callback)
         error = future.exception()

From e2388c65161edefa5d584f6b4958c86f5c1fadb5 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 15 Jan 2025 09:43:30 +0800
Subject: [PATCH 112/200] chore(deps): bump django from 4.2.17 to 4.2.18 in
 /rootfs (#175)

Bumps [django](https://github.com/django/django) from 4.2.17 to 4.2.18.
- [Commits](https://github.com/django/django/compare/4.2.17...4.2.18)

---
updated-dependencies:
- dependency-name: django
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index b017d2244..b65104cf0 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.17
+django==4.2.18
 channels==4.1.0
 aiohttp==3.11.0
 django-cors-headers==4.6.0

From 25a81fb7330aad93b7fdc10ac28ac4125dda0eda Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Mon, 17 Feb 2025 10:35:46 +0800
Subject: [PATCH 113/200] chore(pts): pts replicas not correct (#176)

---
 rootfs/api/models/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index e1e702d44..c01bfecbc 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -563,7 +563,7 @@ def list_deployments(self, *args, **kwargs):
                     'release': labels['version'],
                     'ready': "%s/%s" % (
                         p["status"].get("readyReplicas", 0),
-                        p["status"].get("replicas", 0),
+                        p['spec'].get("replicas", 0),
                     ),
                     'garbage': False if labels['type'] in ptypes else True,
                     'up_to_date': p["status"].get("updatedReplicas", 0),

From 8703034508df4dd744baaf39e64f79985504c695 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 20 Feb 2025 16:46:45 +0800
Subject: [PATCH 114/200] chore(settings): logger simple format (#177)

---
 rootfs/api/settings/production.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 6fecebee2..8acb6eccd 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -189,7 +189,7 @@
             'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s'
         },
         'simple': {
-            'format': '%(levelname)s %(asctime)s %(message)s'
+            'format': '%(levelname)s %(message)s'
         },
     },
     'filters': {

From b571c96da44676c6f7f388e602e3e1286f442c0b Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Mon, 24 Feb 2025 09:50:28 +0800
Subject: [PATCH 115/200] fix(controller): pod CrashLoopBackOff, release should
 be crash (#178)

---
 rootfs/scheduler/resources/pod.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index dc36a383f..a9f733ab7 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -524,14 +524,14 @@ def _handle_pod_errors(self, pod, reason, message):
             'RegistryUnavailable',
             'ErrImageInspect',
             "CreateContainerError",
+            "CrashLoopBackOff",
         ]
         # Image event reason mapping
         event_errors = {
             "Failed": "FailedToPullImage",
             "InspectFailed": "FailedToInspectImage",
             "ErrImageNeverPull": "ErrImageNeverPullPolicy",
-            # Not including this one for now as the message is not useful
-            # "BackOff": "BackOffPullImage",
+            "BackOff": "CrashLoopBackOff",
             # FailedScheduling relates limits
             "FailedScheduling": "FailedScheduling",
         }
@@ -593,7 +593,13 @@ def _handle_pending_pods(self, namespace, labels):
             pods = []
         for pod in pods:
             # only care about pods that are not starting or in the starting phases
-            if pod['status']['phase'] not in ['Pending', 'ContainerCreating']:
+            phase = pod['status']['phase']
+            name = '{}-{}'.format(pod['metadata']['labels']['app'],
+                                  pod['metadata']['labels']['type'])
+            container = self.find_container(name, pod['status']['containerStatuses'])
+            # phase is Running, but state is waiting in CrashLoopBackOff
+            if phase not in ['Pending', 'ContainerCreating'] and \
+                    (phase == 'Running' and 'waiting' not in container['state'].keys()):
                 continue
 
             # Get more information on why a pod is pending

From 235a3fdab46f10721e865d2c1929a7e05dcafb5f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 24 Feb 2025 15:44:31 +0800
Subject: [PATCH 116/200] chore(dryccfile): migrate to dryccfile v2

---
 .../migrations/0022_migration_dryccfile.py    | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 rootfs/api/migrations/0022_migration_dryccfile.py

diff --git a/rootfs/api/migrations/0022_migration_dryccfile.py b/rootfs/api/migrations/0022_migration_dryccfile.py
new file mode 100644
index 000000000..05108fa31
--- /dev/null
+++ b/rootfs/api/migrations/0022_migration_dryccfile.py
@@ -0,0 +1,45 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+
+from django.db import migrations
+from api.models.release import Release
+
+
+def migration_dryccfile(apps, schema_editor):
+    for release in Release.objects.all():
+        build = release.build
+        if not build or not build.dryccfile or "pipeline" in build.dryccfile:
+            continue
+        config, envs = {}, {}
+        if release.config:
+            for value in release.config.values:
+                if "ptype" in value:
+                    if value["ptype"] not in envs:
+                        envs[value["ptype"]] = {}
+                    envs[value["ptype"]][value["name"]] = value["value"]
+                elif "group" in value:
+                    if value["group"] not in envs:
+                        config[value["group"]] = {}
+                    config[value["group"]][value["name"]] = value["value"]
+        pipeline = {}
+        for key, value in build.dryccfile.get('deploy', {}).items():
+            pipeline.update({
+                "%s.yaml" % key: {
+                    "kind": "pipeline",
+                    "ptype": key,
+                    "env": envs.get(key, {}),
+                    "deploy": value,
+                }
+            })
+        build.dryccfile = {"config": config, "pipeline": pipeline}
+        build.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0021_limitplan_runtime_class_name'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_dryccfile),
+    ]

From 85ab5a7a93283446451ffcaa0d9aae330ad46a23 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 25 Feb 2025 17:03:27 +0800
Subject: [PATCH 117/200] chore(charts): change upgrade helm hook

---
 charts/controller/templates/controller-job-upgrade.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
index af78741e2..133e534ad 100644
--- a/charts/controller/templates/controller-job-upgrade.yaml
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -4,8 +4,8 @@ metadata:
   name: drycc-controller-job-upgrade
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
-    helm.sh/hook: post-install,pre-upgrade,pre-rollback
-    helm.sh/hook-delete-policy: before-hook-creation
+    helm.sh/hook: post-install,post-upgrade,post-rollback
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
 spec:
   template:
     spec:

From e899bcfbe25111ab1b8b81fe181551a0e376f50b Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 27 Feb 2025 11:28:15 +0800
Subject: [PATCH 118/200] fix(controller): migrate error

---
 ...{0018_config_values_refs.py => 0017_config_values_refs.py} | 2 +-
 ...es.py => 0018_migration_config_values_and_typed_values.py} | 2 +-
 ...=> 0019_remove_config_typed_values_alter_config_values.py} | 2 +-
 ...ime_class_name.py => 0020_limitplan_runtime_class_name.py} | 2 +-
 ...022_migration_dryccfile.py => 0021_migration_dryccfile.py} | 2 +-
 ...n_deployed_ptypes.py => 0022_migration_deployed_ptypes.py} | 4 ++--
 6 files changed, 7 insertions(+), 7 deletions(-)
 rename rootfs/api/migrations/{0018_config_values_refs.py => 0017_config_values_refs.py} (83%)
 rename rootfs/api/migrations/{0019_migration_config_values_and_typed_values.py => 0018_migration_config_values_and_typed_values.py} (97%)
 rename rootfs/api/migrations/{0020_remove_config_typed_values_alter_config_values.py => 0019_remove_config_typed_values_alter_config_values.py} (87%)
 rename rootfs/api/migrations/{0021_limitplan_runtime_class_name.py => 0020_limitplan_runtime_class_name.py} (85%)
 rename rootfs/api/migrations/{0022_migration_dryccfile.py => 0021_migration_dryccfile.py} (96%)
 rename rootfs/api/migrations/{0017_migration_deployed_ptypes.py => 0022_migration_deployed_ptypes.py} (90%)

diff --git a/rootfs/api/migrations/0018_config_values_refs.py b/rootfs/api/migrations/0017_config_values_refs.py
similarity index 83%
rename from rootfs/api/migrations/0018_config_values_refs.py
rename to rootfs/api/migrations/0017_config_values_refs.py
index abb11eead..f1538341b 100644
--- a/rootfs/api/migrations/0018_config_values_refs.py
+++ b/rootfs/api/migrations/0017_config_values_refs.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0017_migration_deployed_ptypes'),
+        ('api', '0016_remove_route_port_remove_route_ptype_and_more'),
     ]
 
     operations = [
diff --git a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py b/rootfs/api/migrations/0018_migration_config_values_and_typed_values.py
similarity index 97%
rename from rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
rename to rootfs/api/migrations/0018_migration_config_values_and_typed_values.py
index c6eab9215..b269d1844 100644
--- a/rootfs/api/migrations/0019_migration_config_values_and_typed_values.py
+++ b/rootfs/api/migrations/0018_migration_config_values_and_typed_values.py
@@ -35,7 +35,7 @@ def migration_values(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0018_config_values_refs'),
+        ('api', '0017_config_values_refs'),
     ]
 
     operations = [
diff --git a/rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py b/rootfs/api/migrations/0019_remove_config_typed_values_alter_config_values.py
similarity index 87%
rename from rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py
rename to rootfs/api/migrations/0019_remove_config_typed_values_alter_config_values.py
index 78e0eadfd..ee1a0b9fe 100644
--- a/rootfs/api/migrations/0020_remove_config_typed_values_alter_config_values.py
+++ b/rootfs/api/migrations/0019_remove_config_typed_values_alter_config_values.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0019_migration_config_values_and_typed_values'),
+        ('api', '0018_migration_config_values_and_typed_values'),
     ]
 
     operations = [
diff --git a/rootfs/api/migrations/0021_limitplan_runtime_class_name.py b/rootfs/api/migrations/0020_limitplan_runtime_class_name.py
similarity index 85%
rename from rootfs/api/migrations/0021_limitplan_runtime_class_name.py
rename to rootfs/api/migrations/0020_limitplan_runtime_class_name.py
index 10e334f95..c473af4e0 100644
--- a/rootfs/api/migrations/0021_limitplan_runtime_class_name.py
+++ b/rootfs/api/migrations/0020_limitplan_runtime_class_name.py
@@ -6,7 +6,7 @@
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0020_remove_config_typed_values_alter_config_values'),
+        ('api', '0019_remove_config_typed_values_alter_config_values'),
     ]
 
     operations = [
diff --git a/rootfs/api/migrations/0022_migration_dryccfile.py b/rootfs/api/migrations/0021_migration_dryccfile.py
similarity index 96%
rename from rootfs/api/migrations/0022_migration_dryccfile.py
rename to rootfs/api/migrations/0021_migration_dryccfile.py
index 05108fa31..fc5dad40f 100644
--- a/rootfs/api/migrations/0022_migration_dryccfile.py
+++ b/rootfs/api/migrations/0021_migration_dryccfile.py
@@ -37,7 +37,7 @@ def migration_dryccfile(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0021_limitplan_runtime_class_name'),
+        ('api', '0020_limitplan_runtime_class_name'),
     ]
 
     operations = [
diff --git a/rootfs/api/migrations/0017_migration_deployed_ptypes.py b/rootfs/api/migrations/0022_migration_deployed_ptypes.py
similarity index 90%
rename from rootfs/api/migrations/0017_migration_deployed_ptypes.py
rename to rootfs/api/migrations/0022_migration_deployed_ptypes.py
index 25d2f7831..b5a9768ac 100644
--- a/rootfs/api/migrations/0017_migration_deployed_ptypes.py
+++ b/rootfs/api/migrations/0022_migration_deployed_ptypes.py
@@ -20,9 +20,9 @@ def migration_deployed_ptypes(apps, schema_editor):
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('api', '0016_remove_route_port_remove_route_ptype_and_more'),
+        ('api', '0021_migration_dryccfile'),
     ]
 
     operations = [
         migrations.RunPython(migration_deployed_ptypes),
-    ]
+    ]
\ No newline at end of file

From d016d6cbbacde1edf5db3b39edd35dc5b064f983 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 27 Feb 2025 17:32:56 +0800
Subject: [PATCH 119/200] fix(controller): secret mistakenly deleted when force
 deploy (#179)

---
 rootfs/api/models/release.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index a1c3ab47f..fb07a810e 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -293,8 +293,9 @@ def clean(self, ptypes=None):
             replica_sets = []
         for replica_set in replica_sets:
             current_version_name = replica_set['metadata']['labels']['version']
-            # skip the latest release
-            if current_version_name == self.version_name:
+            # skip the later release
+            current_version = int(current_version_name[1:])
+            if current_version >= self.version:
                 continue
 
             # aggregate versions together to removal all at once
@@ -313,9 +314,9 @@ def clean(self, ptypes=None):
         # handle Deployments specific cleanups
         self._cleanup_deployment_secrets_and_configs(self.app.id, ptypes)
         # Remove stray pods
-        self._cleanup_stray_pods(self.app.id, ptypes, self.version_name)
+        self._cleanup_stray_pods(self.app.id, ptypes, self.version)
 
-    def _cleanup_stray_pods(self, namespace, ptypes, latest_version_name):
+    def _cleanup_stray_pods(self, namespace, ptypes, latest_version):
         labels = {'heritage': 'drycc'}
         if ptypes is not None:
             labels['type__in'] = ptypes
@@ -326,9 +327,9 @@ def _cleanup_stray_pods(self, namespace, ptypes, latest_version_name):
             if self.scheduler().pod.deleted(pod):
                 continue
 
-            current_version_name = pod['metadata']['labels']['version']
-            # skip the latest release
-            if current_version_name == latest_version_name:
+            current_version = int(pod['metadata']['labels']['version'][1:])
+            # skip the later release
+            if current_version >= latest_version:
                 continue
 
             try:
@@ -355,8 +356,11 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, ptypes=None):
         if not replicasets:
             replicasets = []
         for replicaset in replicasets:
+            replica_version = int(replicaset['metadata']['labels']['version'][1:])
+            # only remove the older releases secret
             if (
                 'version' not in replicaset['metadata']['labels'] or
+                replica_version < self.version or
                 replicaset['metadata']['labels']['version'] in version_names
             ):
                 continue

From edfeef9be820356833d07c63f844eba961b6508c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 26 Mar 2025 11:46:54 +0800
Subject: [PATCH 120/200] chore(deps): bump django from 4.2.18 to 4.2.20 in
 /rootfs (#180)

Bumps [django](https://github.com/django/django) from 4.2.18 to 4.2.20.
- [Commits](https://github.com/django/django/compare/4.2.18...4.2.20)

---
updated-dependencies:
- dependency-name: django
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index b65104cf0..8e1e8249a 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.18
+django==4.2.20
 channels==4.1.0
 aiohttp==3.11.0
 django-cors-headers==4.6.0

From 636dceb65a8d1bf95a6ee240cdf93ab89f77b883 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 27 Mar 2025 10:20:55 +0800
Subject: [PATCH 121/200] chore(controller): allow re attach to update and some
 error (#181)

* fix(timeouts): timeouts set error

* fix(tls): tls force enable error

* chore(certs): allow re attach to update

* chore(controller): https_enforced when has domains

* chore(controller): pod pending has not containerStatuses
---
 rootfs/api/models/certificate.py                |  4 +---
 rootfs/api/models/config.py                     | 15 +++++++++++++++
 rootfs/api/models/gateway.py                    |  5 +++--
 rootfs/api/tests/test_certificate_use_case_4.py |  8 ++++----
 rootfs/scheduler/resources/pod.py               |  8 ++++----
 5 files changed, 27 insertions(+), 13 deletions(-)

diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index d97d0c16f..b1f9cd5b1 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -13,7 +13,7 @@
 from django.contrib.auth import get_user_model
 from rest_framework.exceptions import ValidationError
 from api.utils import validate_label
-from api.exceptions import AlreadyExists, ServiceUnavailable
+from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 from .base import AuditedModel
 from .domain import Domain
@@ -192,8 +192,6 @@ def delete(self, *args, **kwargs):
     def attach(self, *args, **kwargs):
         # add the certificate to the domain
         domain = get_object_or_404(Domain, domain=kwargs['domain'])
-        if domain.certificate is not None:
-            raise AlreadyExists("Domain already has a certificate attached to it")
         # create in kubernetes
         self.attach_in_kubernetes(domain)
         domain.certificate = self
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 7be9ed191..4c7742f31 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -265,3 +265,18 @@ def _update_limits(self, previous_config, replace_ptypes=[]):
                         "the %s has already been used and cannot be deleted" % ptype)
         self._merge_data('limits', data, new_data)
         setattr(self, 'limits', data)
+
+    def _update_termination_grace_period(self, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, 'termination_grace_period', {}).copy().items()  # noqa
+            if k not in replace_ptypes
+        }
+        new_data = getattr(self, 'termination_grace_period', {}).copy()
+        # check procfile
+        for ptype, value in new_data.items():
+            if value is None:
+                if ptype in self.app.ptypes:
+                    raise UnprocessableEntity(
+                        "the %s has already been used and cannot be deleted" % ptype)
+        self._merge_data('termination_grace_period', data, new_data)
+        setattr(self, 'termination_grace_period', data)
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index af1cad24e..6aa15cccb 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -222,7 +222,7 @@ def cleaned_rules(self):
     @property
     def tls_force_hostnames(self):
         tls = self.app.tls_set.latest()
-        q = Q(ptype__int=[s.ptype for s in self.services])
+        q = Q(ptype__in=[s.ptype for s in self.services])
         if not tls.certs_auto_enabled:
             q &= Q(certificate__isnull=False)
         domains = self.app.domain_set.filter(q)
@@ -260,7 +260,8 @@ def refresh_to_k8s(self):
         if self.routable:
             parent_refs, http_parent_refs = self._get_all_parent_refs()
             tls = self.app.tls_set.latest()
-            if tls.https_enforced and self.kind == "HTTPRoute":
+            # requestRedirect only when has tls or certs
+            if tls.https_enforced and self.kind == "HTTPRoute" and self.tls_force_hostnames:
                 self._https_enforced_to_k8s(http_parent_refs)
             elif self.kind == "HTTPRoute":
                 parent_refs.extend(http_parent_refs)
diff --git a/rootfs/api/tests/test_certificate_use_case_4.py b/rootfs/api/tests/test_certificate_use_case_4.py
index 9bfd62c6f..f12b7634b 100644
--- a/rootfs/api/tests/test_certificate_use_case_4.py
+++ b/rootfs/api/tests/test_certificate_use_case_4.py
@@ -168,8 +168,8 @@ def test_certificate_attach_overwrite(self):
             '{}/{}/domain/'.format(self.url, 'bar-com'),
             {'domain': 'foo.com'}
         )
-        # Should be a 409 Conflict since it already existed
-        self.assertEqual(response.status_code, 409)
+        # allow re attach to update
+        self.assertEqual(response.status_code, 201)
 
         # Assert that domain and cert are still the original
         response = self.client.get(
@@ -180,7 +180,7 @@ def test_certificate_attach_overwrite(self):
         expected = {
             'name': 'foo-com',
             'common_name': 'foo.com',
-            'domains': ['foo.com']
+            'domains': []
         }
         for key, value in list(expected.items()):
             self.assertEqual(
@@ -197,7 +197,7 @@ def test_certificate_attach_overwrite(self):
         expected = {
             'name': 'bar-com',
             'common_name': 'bar.com',
-            'domains': []
+            'domains': ['foo.com']
         }
         for key, value in list(expected.items()):
             self.assertEqual(
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index a9f733ab7..13af05245 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -413,7 +413,7 @@ def readiness_status(self, pod):
         """Check if the pod container have passed the readiness probes"""
         name = '{}-{}'.format(pod['metadata']['labels']['app'], pod['metadata']['labels']['type'])
         # find the right container in case there are many on the pod
-        container = self.find_container(name, pod['status']['containerStatuses'])
+        container = self.find_container(name, pod['status'].get('containerStatuses', []))
         if container is None:
             # Seems like the most sensible default
             return 'Unknown'
@@ -466,7 +466,7 @@ def pending_status(self, pod):
 
         name = '{}-{}'.format(pod['metadata']['labels']['app'], pod['metadata']['labels']['type'])
         # find the right container in case there are many on the pod
-        container = self.pod.find_container(name, pod['status']['containerStatuses'])
+        container = self.find_container(name, pod['status'].get('containerStatuses', []))
         if container is None:
             # Return Pending if nothing else can be found
             return 'Pending', ''
@@ -596,7 +596,7 @@ def _handle_pending_pods(self, namespace, labels):
             phase = pod['status']['phase']
             name = '{}-{}'.format(pod['metadata']['labels']['app'],
                                   pod['metadata']['labels']['type'])
-            container = self.find_container(name, pod['status']['containerStatuses'])
+            container = self.find_container(name, pod['status'].get('containerStatuses', []))
             # phase is Running, but state is waiting in CrashLoopBackOff
             if phase not in ['Pending', 'ContainerCreating'] and \
                     (phase == 'Running' and 'waiting' not in container['state'].keys()):
@@ -748,7 +748,7 @@ def _handle_not_ready_pods(self, namespace, labels):
 
             name = '{}-{}'.format(pod['metadata']['labels']['app'], pod['metadata']['labels']['type'])  # noqa
             # find the right container in case there are many on the pod
-            container = self.find_container(name, pod['status']['containerStatuses'])
+            container = self.find_container(name, pod['status'].get('containerStatuses', []))
             if container is None or container['ready'] == 'true':
                 continue
 

From 9d31b369275696dbe9bbb7cb6c780ee5b91381fc Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 27 Mar 2025 10:37:10 +0800
Subject: [PATCH 122/200] chore(controller): add oss mount support

---
 charts/controller/templates/_helpers.tpl      |  83 ++++++++-
 .../controller-celery-deloyment.yaml          |   8 +
 .../templates/controller-clusterrole.yaml     |   3 +
 .../templates/controller-configmap.yaml       |  18 ++
 charts/controller/values.yaml                 |   3 +
 .../0023_migration_k8s_alter_volume_type.py   |  26 +++
 rootfs/api/models/app.py                      |   8 +-
 rootfs/api/models/volume.py                   | 172 +++++++++++++-----
 rootfs/api/serializers/__init__.py            |   6 +-
 rootfs/api/serializers/schemas/volumes.py     |  13 +-
 rootfs/api/settings/production.py             |  24 +++
 rootfs/api/settings/testing.py                |  31 ++++
 rootfs/api/tests/test_volume.py               |   1 -
 rootfs/api/views.py                           |  13 +-
 rootfs/scheduler/resources/pv.py              |  73 ++++++++
 rootfs/scheduler/resources/pvc.py             |  13 +-
 16 files changed, 419 insertions(+), 76 deletions(-)
 create mode 100644 rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py
 create mode 100644 rootfs/scheduler/resources/pv.py

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index c297db592..9a9774d8f 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -342,4 +342,85 @@ kubelet_volume_stats_capacity_bytes: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim, job]
-{{- end }}
\ No newline at end of file
+{{- end }}
+
+
+{{/* Generate controller config default secrets template */}}
+{{ define "controller.config.defaultSecretTemplate" }}
+oss:
+  data:
+    key_id: $access_key
+    access_key: $secret_key
+{{- end }}
+
+{{/* Generate controller config default volume template */}}
+{{ define "controller.config.defaultVolumeTemplate" }}
+nfs:
+  spec:
+    capacity:
+      storage: 65535Gi
+    accessModes:
+    - ReadWriteMany
+    claimRef:
+      namespace: $namespace
+      name: $volume_claim_name
+    nfs:
+      server: $server
+      path: $path
+    mountOptions:
+    - nfsvers=4.2
+oss:
+  spec:
+    capacity:
+      storage: 65535Gi
+    accessModes:
+    - ReadWriteMany
+    storageClassName: ''
+    claimRef:
+      namespace: $namespace
+      name: $volume_claim_name
+    mountOptions:
+    - allow-delete
+    - force-path-style
+    - endpoint-url $server
+    csi:
+      driver: s3.csi.aws.com
+      volumeHandle: $volume_handle
+      volumeAttributes:
+        bucketName: $bucket
+        authenticationSource: secrets
+      nodePublishSecretRef:
+        name: $secret_name
+        namespace: $namespace
+{{- end }}
+
+{{/* Generate controller config default volume claim template */}}
+{{ define "controller.config.defaultVolumeClaimTemplate" }}
+csi:
+  spec:
+    accessModes:
+    - ReadWriteMany
+    storageClassName: $storage_class
+    resources:
+      requests:
+        storage: $size
+    volumeMode: Filesystem
+nfs:
+  spec:
+    accessModes:
+    - ReadWriteMany
+    storageClassName: ''
+    resources:
+      requests:
+        storage: $size
+    volumeName: $volume_name
+oss:
+  spec:
+    accessModes:
+    - ReadWriteMany
+    storageClassName: ''
+    resources:
+      requests:
+        storage: $size
+    volumeName: $volume_name
+{{- end }}
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/controller-celery-deloyment.yaml
index d2d1f5cc0..10972d934 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/controller-celery-deloyment.yaml
@@ -53,3 +53,11 @@ spec:
         {{- end }}
         {{- include "controller.limits" $ | indent 8 }}
         {{- include "controller.envs" $ | indent 8 }}
+        volumeMounts:
+        - name: controller-config
+          readOnly: false
+          mountPath: /etc/controller
+      volumes:
+      - name: controller-config
+        configMap:
+          name: controller-config
\ No newline at end of file
diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index 0084190cd..503e0c61c 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -70,6 +70,9 @@ rules:
 - apiGroups: ["apps"]
   resources: ["replicasets"]
   verbs: ["get", "list", "watch", "delete"]
+- apiGroups: [""]
+  resources: ["persistentvolumes"]
+  verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
 - apiGroups: [""]
   resources: ["persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
diff --git a/charts/controller/templates/controller-configmap.yaml b/charts/controller/templates/controller-configmap.yaml
index 8b5cb5b35..542cba653 100644
--- a/charts/controller/templates/controller-configmap.yaml
+++ b/charts/controller/templates/controller-configmap.yaml
@@ -23,3 +23,21 @@ data:
     {{- else}}
     {{- include "controller.config.defaultLimitPlans" . | fromYamlArray | toPrettyJson | nindent 4 }}
     {{- end }}
+  secret-template.json: |
+    {{- if .Values.config.secretTemplate }}
+    {{- (tpl .Values.config.secretTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultSecretTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{- end }}
+  volume-template.json: |
+    {{- if .Values.config.volumeTemplate }}
+    {{- (tpl .Values.config.volumeTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultVolumeTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{- end }}
+  volume-claim-template.json: |
+    {{- if .Values.config.volumeClaimTemplate }}
+    {{- (tpl .Values.config.volumeClaimTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultVolumeClaimTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{- end }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index d0a4e90e8..795f7239f 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -73,6 +73,9 @@ config:
   metrics: ""
   limitSpecs: ""
   limitPlans: ""
+  secretTemplate: ""
+  volumeTemplate: ""
+  volumeClaimTemplate: ""
 
 # Service
 service:
diff --git a/rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py b/rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py
new file mode 100644
index 000000000..0adac79fa
--- /dev/null
+++ b/rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py
@@ -0,0 +1,26 @@
+# Generated by Django 4.2.18 on 2025-03-26 02:44
+
+from django.db import migrations, models
+from api.models.volume import Volume
+
+
+def save_volume_to_k8s(apps, schema_editor):
+    for volume in Volume.objects.all():
+        if volume.type == 'nfs':
+            volume.save_to_k8s()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0022_migration_deployed_ptypes'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='volume',
+            name='type',
+            field=models.CharField(choices=[('csi', 'container storage interface'), ('nfs', 'network file system'), ('oss', 'object storage service file')], default='csi'),
+        ),
+        migrations.RunPython(save_volume_to_k8s),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index c01bfecbc..c3464127d 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1175,12 +1175,8 @@ def _get_volumes_and_mounts(self, ptype, volumes):
         k8s_volumes, k8s_volume_mounts = [], []
         if volumes:
             for volume in volumes:
-                k8s_volume = {"name": volume.name}
-                if volume.type == "csi":
-                    k8s_volume.update({"persistentVolumeClaim": {"claimName": volume.name}})
-                else:
-                    k8s_volume.update(volume.parameters)
-                k8s_volumes.append(k8s_volume)
+                k8s_volumes.append(
+                    {"name": volume.name, "persistentVolumeClaim": {"claimName": volume.name}})
                 k8s_volume_mounts.append(
                     {"name": volume.name, "mountPath": volume.path.get(ptype)})
         return k8s_volumes, k8s_volume_mounts
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 666e386d2..6821fb037 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -1,9 +1,14 @@
 import logging
+import string
+import copy
+import json
+import uuid
+import hashlib
 from django.db import models, transaction
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from api.utils import unit_to_bytes, validate_label
-from api.exceptions import DryccException, ServiceUnavailable, AlreadyExists
+from api.exceptions import DryccException, ServiceUnavailable
 from scheduler import KubeException
 from .base import UuidAuditedModel
 
@@ -15,6 +20,7 @@ class Volume(UuidAuditedModel):
     TYPE_CHOICES = (
         ("csi", "container storage interface"),
         ("nfs", "network file system"),
+        ("oss", "object storage service file"),
     )
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
@@ -24,11 +30,29 @@ class Volume(UuidAuditedModel):
     type = models.CharField(default=TYPE_CHOICES[0][0], choices=TYPE_CHOICES)
     parameters = models.JSONField(default=dict)
 
+    @property
+    def pv_name(self):
+        md5 = hashlib.md5()
+        md5.update(f"{self.app.id}-{self.type}-{self.name}".encode("utf-8"))
+        hexdigest = md5.hexdigest()
+        return "pvc-{}".format("-".join([
+            hexdigest[:8],
+            hexdigest[8:12],
+            hexdigest[12:16],
+            hexdigest[16:20],
+            hexdigest[20:]
+        ]))
+
+    @property
+    def secret_name(self):
+        return self.pv_name
+
     @transaction.atomic
     def save(self, *args, **kwargs):
+        if self.type not in settings.DRYCC_VOLUME_CLAIM_TEMPLATE:
+            raise DryccException(f'Volume type {self.type} is not supported.')
         # Attach volume, updates k8s
-        if self.type == "csi" and self.created == self.updated:
-            self._create_pvc()
+        self.save_to_k8s()
         # check path
         self.check_path()
         # Save to DB
@@ -37,31 +61,10 @@ def save(self, *args, **kwargs):
     @transaction.atomic
     def delete(self, *args, **kwargs):
         # Deatch volume, updates k8s
-        if self.type == "csi":
-            self._delete_pvc()
+        self.delete_from_k8s()
         # Delete from DB
         return super(Volume, self).delete(*args, **kwargs)
 
-    @transaction.atomic
-    def expand(self, size):
-        if self.type == "csi":
-            if unit_to_bytes(size) < unit_to_bytes(self.size):
-                raise DryccException('Shrink volume is not supported.')
-            self.size = size
-            self.save()
-            try:
-                kwargs = {
-                    "size": self._format_size(self.size),
-                    "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
-                }
-                self.scheduler().pvc.patch(self.app.id, self.name, **kwargs)
-            except KubeException as e:
-                msg = 'There was a problem expand the volume ' \
-                        '{} for {}'.format(self.name, self.app_id)
-                raise ServiceUnavailable(msg) from e
-        else:
-            raise DryccException(f'{self.type} volume is not support expand.')
-
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this service.
 
@@ -90,16 +93,6 @@ def to_measurements(self, timestamp: float):
     def __str__(self):
         return self.name
 
-    @staticmethod
-    def _format_size(size):
-        """ Format volume limit value """
-        if size[-2:-1].isalpha() and size[-1].isalpha():
-            size = size[:-1]
-
-        if size[-1].isalpha():
-            size = size.upper() + "i"
-        return size
-
     def check_path(self, path=None):
         other_volumes = self.app.volume_set.exclude(name=self.name)
         type_paths = {}  # {'type1':[path1,path2], tyep2:[path3,path4]}
@@ -116,22 +109,84 @@ def check_path(self, path=None):
             self.log(msg, logging.ERROR)
             raise DryccException(msg)
 
-    def _create_pvc(self):
+    def save_to_k8s(self):
+        if self.type in settings.DRYCC_SECRET_TEMPLATE:
+            self._save_secret()
+        if self.type in settings.DRYCC_VOLUME_TEMPLATE:
+            self._save_pv()
+        self._save_pvc()
+
+    def delete_from_k8s(self):
+        if self.type in settings.DRYCC_SECRET_TEMPLATE:
+            self._delete_secret()
+        if self.type in settings.DRYCC_VOLUME_TEMPLATE:
+            self._delete_pv()
+        self._delete_pvc()
+
+    @staticmethod
+    def _format_size(size):
+        """ Format volume limit value """
+        if size[-2:-1].isalpha() and size[-1].isalpha():
+            size = size[:-1]
+
+        if size[-1].isalpha():
+            size = size.upper() + "i"
+        return size
+
+    def _save_pv(self):
+        kwds = copy.deepcopy(self.parameters.get(self.type, {}))
+        kwds.update({
+            "volume_claim_name": self.name,
+            "namespace": self.app.id,
+            "secret_name": self.secret_name,
+            "volume_handle": "%s" % uuid.uuid4(),
+        })
+        t = string.Template(json.dumps(settings.DRYCC_VOLUME_TEMPLATE.get(self.type)))
+        kwargs = json.loads(t.safe_substitute(**kwds))
+        try:
+            self.scheduler().pv.get(self.pv_name)
+            msg = "Volume {} already exists".format(self.pv_name)
+            self.log(msg, logging.INFO)
+            if "csi" in kwargs["spec"]:
+                del kwargs["spec"]["csi"]  # Fix: 422 unprocessable Entity
+            self.scheduler().pv.patch(self.pv_name, **kwargs)
+        except KubeException as e:
+            logger.info(e)
+            try:
+                self.scheduler().pv.create(self.pv_name, **kwargs)
+            except KubeException as e:
+                msg = 'There was a problem creating the volume ' \
+                      '{} for {}'.format(self.pv_name, self.app_id)
+                raise ServiceUnavailable(msg) from e
+
+    def _delete_pv(self):
+        try:
+            # We raise an exception when a volume doesn't exist
+            self.scheduler().pv.get(self.pv_name)
+            self.scheduler().pv.delete(self.pv_name)
+        except KubeException as e:
+            logger.exception(e)
+
+    def _save_pvc(self):
+        kwds = copy.deepcopy(self.parameters)
+        kwds.update({
+            "size": self._format_size(self.size),
+            "volume_name": self.pv_name,
+            "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
+        })
+        t = string.Template(json.dumps(settings.DRYCC_VOLUME_CLAIM_TEMPLATE.get(self.type)))
+        kwargs = json.loads(t.safe_substitute(**kwds))
         try:
             self.scheduler().pvc.get(self.app.id, self.name)
-            err = "Volume {} already exists in this namespace".format(self.name)
-            self.log(err, logging.INFO)
-            raise AlreadyExists(err)
+            msg = "Volume claim {} already exists in this namespace".format(self.name)
+            self.log(msg, logging.INFO)
+            self.scheduler().pvc.patch(self.app.id, self.name, **kwargs)
         except KubeException as e:
             logger.info(e)
             try:
-                kwargs = {
-                    "size": self._format_size(self.size),
-                    "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
-                }
                 self.scheduler().pvc.create(self.app.id, self.name, **kwargs)
             except KubeException as e:
-                msg = 'There was a problem creating the volume ' \
+                msg = 'There was a problem creating the volume claim ' \
                       '{} for {}'.format(self.name, self.app_id)
                 raise ServiceUnavailable(msg) from e
 
@@ -141,8 +196,33 @@ def _delete_pvc(self):
             self.scheduler().pvc.get(self.app.id, self.name)
             self.scheduler().pvc.delete(self.app.id, self.name)
         except KubeException as e:
-            raise ServiceUnavailable("Could not delete volume {} for application \
-                {}".format(self.name, self.app_id)) from e  # noqa
+            logger.exception(e)
+
+    def _save_secret(self):
+        kwds = copy.deepcopy(self.parameters.get(self.type, {}))
+        t = string.Template(json.dumps(settings.DRYCC_SECRET_TEMPLATE.get(self.type)))
+        kwargs = json.loads(t.safe_substitute(**kwds))
+        try:
+            self.scheduler().secret.get(self.app.id, self.secret_name)
+            msg = "Secret {} already exists".format(self.secret_name)
+            self.log(msg, logging.INFO)
+            self.scheduler().secret.patch(self.app.id, self.secret_name, **kwargs)
+        except KubeException as e:
+            logger.info(e)
+            try:
+                self.scheduler().secret.create(self.app.id, self.secret_name, **kwargs)
+            except KubeException as e:
+                msg = 'There was a problem creating the volume ' \
+                      '{} for {}'.format(self.secret_name, self.app_id)
+                raise ServiceUnavailable(msg) from e
+
+    def _delete_secret(self):
+        try:
+            # We raise an exception when a volume doesn't exist
+            self.scheduler().secret.get(self.app.id, self.secret_name)
+            self.scheduler().secret.delete(self.app.id, self.secret_name)
+        except KubeException as e:
+            logger.exception(e)
 
     class Meta:
         get_latest_by = 'created'
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index e1e3c6b6b..6aebc2b65 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -17,6 +17,7 @@
 from api import models
 from api.utils import validate_json
 from api.exceptions import DryccException
+from api.models.volume import Volume
 from api.models.base import PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
 from .schemas import rules
@@ -49,7 +50,7 @@
 TERMINATION_GRACE_PERIOD_MATCH = re.compile(r'^[0-9]*$')
 TERMINATION_GRACE_PERIOD_MISMATCH_MSG = (
     "Termination Grace Period format: %s" % TERMINATION_GRACE_PERIOD_MATCH.pattern)
-VOLUME_TYPE_MATCH = re.compile(r'^(csi|nfs)$')
+VOLUME_TYPE_MATCH = re.compile(r'^(%s)$' % "|".join([v[0] for v in Volume.TYPE_CHOICES]))
 VOLUME_TYPE_MISMATCH_MSG = "Volume type pattern: %s" % VOLUME_TYPE_MATCH.pattern
 VOLUME_SIZE_MATCH = re.compile(r'^(?P<volume>([1-9][0-9]*[gG]))$', re.IGNORECASE)
 VOLUME_SIZE_MISMATCH_MSG = (
@@ -627,10 +628,7 @@ class VolumeSerializer(serializers.ModelSerializer):
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     owner = serializers.ReadOnlyField(source='owner.username')
-    name = serializers.CharField()
-    size = serializers.CharField(required=False)
     path = JSONFieldSerializer(required=False, binary=True)
-    type = serializers.CharField(required=False)
     parameters = serializers.JSONField(required=False)
 
     class Meta:
diff --git a/rootfs/api/serializers/schemas/volumes.py b/rootfs/api/serializers/schemas/volumes.py
index 2bcb9842d..1b44509de 100644
--- a/rootfs/api/serializers/schemas/volumes.py
+++ b/rootfs/api/serializers/schemas/volumes.py
@@ -7,9 +7,18 @@
             "properties": {
                 "server": {"type": "string"},
                 "path": {"type": "string"},
-                "readOnly": {"type": "boolean"},
             },
-            "required": ["server", "path", "readOnly"],
+            "required": ["server", "path"],
+        },
+        "oss": {
+            "type": "object",
+            "properties": {
+                "bucket": {"type": "string"},
+                "server": {"type": "string"},
+                "access_key": {"type": "string"},
+                "secret_key": {"type": "string"},
+            },
+            "required": ["bucket", "server", "access_key", "secret_key"],
         },
     },
 }
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 8acb6eccd..a05c59142 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -282,6 +282,30 @@
 DRYCC_METRICS_EXPIRY = int(os.environ.get('DRYCC_METRICS_EXPIRY', '20'))
 DRYCC_METRICS_INTERVAL = os.environ.get('DRYCC_METRICS_INTERVAL', '5m')
 
+# drycc secret template
+DRYCC_SECRET_TEMPLATE = {}
+DRYCC_SECRET_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_SECRET_TEMPLATE_PATH', '/etc/controller/secret-template.json')
+if os.path.exists(DRYCC_SECRET_TEMPLATE_PATH):
+    with open(DRYCC_SECRET_TEMPLATE_PATH) as fd:
+        DRYCC_SECRET_TEMPLATE = json.load(fd)
+
+# drycc volume template
+DRYCC_VOLUME_TEMPLATE = {}
+DRYCC_VOLUME_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_VOLUME_TEMPLATE_PATH', '/etc/controller/volume-template.json')
+if os.path.exists(DRYCC_VOLUME_TEMPLATE_PATH):
+    with open(DRYCC_VOLUME_TEMPLATE_PATH) as fd:
+        DRYCC_VOLUME_TEMPLATE = json.load(fd)
+
+# drycc volume claim template
+DRYCC_VOLUME_CLAIM_TEMPLATE = {}
+DRYCC_VOLUME_CLAIM_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_VOLUME_CLAIM_TEMPLATE_PATH', '/etc/controller/volume-claim-template.json')
+if os.path.exists(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH):
+    with open(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH) as fd:
+        DRYCC_VOLUME_CLAIM_TEMPLATE = json.load(fd)
+
 # security keys and auth tokens
 random_secret = 'CHANGEME_sapm$s%upvsw5l_zuy_&29rkywd^78ff(qi*#@&*^'
 SECRET_KEY = os.environ.get('DRYCC_SECRET_KEY', random_secret)
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index 48cfab261..2ed583b93 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -60,3 +60,34 @@ def __getitem__(self, item):
 # WORKFLOW_MANAGER_URL = "http://127.0.0.1:8000"
 WORKFLOW_MANAGER_ACCESS_KEY = "1234567890"
 WORKFLOW_MANAGER_SECRET_KEY = "1234567890"
+
+DRYCC_VOLUME_CLAIM_TEMPLATE = {
+    "csi": {
+        "spec": {
+            "accessModes": [
+                "ReadWriteMany"
+            ],
+            "resources": {
+                "requests": {
+                    "storage": "$size"
+                }
+            },
+            "storageClassName": "$storage_class",
+            "volumeMode": "Filesystem"
+        }
+    },
+    "nfs": {
+        "spec": {
+            "accessModes": [
+                "ReadWriteMany"
+            ],
+            "resources": {
+                "requests": {
+                    "storage": "$size"
+                }
+            },
+            "storageClassName": "",
+            "volumeName": "$volume_name"
+        }
+    }
+}
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index d95301a59..3cdaaf8e7 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -161,7 +161,6 @@ def test_volume_expand(self, mock_requests):
                     'nfs': {
                         'server': 'test.drycc.cc',
                         'path': '/',
-                        'readOnly': False,
                     }
                 }
             }
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 9e4681cbd..734489102 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -904,10 +904,15 @@ def get_object(self):
                                  name=self.kwargs['name'])
 
     def expand(self, request, **kwargs):
-        volume = self.get_object()
-        volume.expand(request.data['size'])
-        serializer = self.get_serializer(volume, many=False)
-        return Response(serializer.data)
+        size, volume = request.data['size'], self.get_object()
+        if volume.type == "csi":
+            if utils.unit_to_bytes(request.data['size']) < utils.unit_to_bytes(volume.size):
+                raise DryccException('Shrink volume is not supported.')
+            volume.size = size
+            volume.save()
+            serializer = self.get_serializer(volume, many=False)
+            return Response(serializer.data)
+        raise DryccException(f'{volume.type} volume is not support expand.')
 
     def destroy(self, request, **kwargs):
         volume = self.get_object()
diff --git a/rootfs/scheduler/resources/pv.py b/rootfs/scheduler/resources/pv.py
new file mode 100644
index 000000000..afb8d8b10
--- /dev/null
+++ b/rootfs/scheduler/resources/pv.py
@@ -0,0 +1,73 @@
+import json
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class PersistentVolume(Resource):
+    short_name = 'pv'
+
+    def manifest(self, name, version=None, **kwargs):
+        labels = {
+            'heritage': 'drycc',
+        }
+        data = {
+            "apiVersion": self.api_version,
+            "kind": "PersistentVolume",
+            "metadata": {
+                "name": name,
+                'labels': labels
+            },
+        }
+        data.update(kwargs)
+        if version:
+            data["metadata"]["resourceVersion"] = version
+        return data
+
+    def get(self, name=None):
+        """
+        Fetch a single persistentvolume or a list of persistentvolumes
+        """
+        if name is not None:
+            url = self.api('/persistentvolumes/{}', name)
+            message = 'get persistentvolume ' + name
+        else:
+            url = self.api('/persistentvolumes')
+            message = 'get persistentvolumes'
+        response = self.http_get(url)
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response, message)
+        return response
+
+    def create(self, name, **kwargs):
+        """
+        Create persistentvolume
+        """
+        url = self.api('/persistentvolumes')
+        data = self.manifest(name, **kwargs)
+        response = self.http_post(url, json=data)
+        if not response.status_code == 201:
+            raise KubeHTTPException(
+                response,
+                "create persistentvolume"
+            )
+        return response
+
+    def patch(self, name, **kwargs):
+        url = self.api('/persistentvolumes/{}', name)
+        data = self.manifest(name, **kwargs)
+        response = self.http_patch(url, json=data, headers={"Content-Type": "application/merge-patch+json"})  # noqa
+        if self.unhealthy(response.status_code):
+            self.log('template used: {}'.format(json.dumps(data, indent=4)), 'DEBUG')
+            raise KubeHTTPException(response, 'update persistentvolume "{}"', name)
+        return response
+
+    def delete(self, name):
+        """
+        Delete persistentvolume
+        """
+        url = self.api('/persistentvolumes/{}', name)
+        response = self.http_delete(url)
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response,
+                                    'delete persistentvolume ' + name)
+        return response
diff --git a/rootfs/scheduler/resources/pvc.py b/rootfs/scheduler/resources/pvc.py
index 4ccfbf629..5ce7df202 100644
--- a/rootfs/scheduler/resources/pvc.py
+++ b/rootfs/scheduler/resources/pvc.py
@@ -17,20 +17,9 @@ def manifest(self, namespace, name, version=None, **kwargs):
                 "name": name,
                 "namespace": namespace,
                 'labels': labels
-            },
-            "spec": {
-                "accessModes": [
-                    "ReadWriteMany"
-                ],
-                "resources": {
-                    "requests": {
-                        "storage": kwargs.get('size')
-                    },
-                },
-                "storageClassName": kwargs.get("storage_class"),
-                "volumeMode": "Filesystem",
             }
         }
+        data.update(kwargs)
         if version:
             data["metadata"]["resourceVersion"] = version
         return data

From d6f358ae97c642c3c845c78e4cb151b8849a48af Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 3 Apr 2025 13:10:26 +0800
Subject: [PATCH 123/200] chore(controller): bump version

---
 rootfs/Dockerfile           |  2 +-
 rootfs/Dockerfile.test      |  6 +++---
 rootfs/dev_requirements.txt |  4 ++--
 rootfs/requirements.txt     | 24 ++++++++++++------------
 4 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile
index 4dbaea4f4..4f151964c 100644
--- a/rootfs/Dockerfile
+++ b/rootfs/Dockerfile
@@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME}
 ENV DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/workspace \
-  PYTHON_VERSION="3.12"
+  PYTHON_VERSION="3.13"
 
 RUN groupadd drycc --gid ${DRYCC_GID} \
   && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR}
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index 099f9444a..f8e6d67fb 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -7,9 +7,9 @@ COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt
 COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
 
 ENV PGDATA="/opt/drycc/postgresql/data" \
-  PYTHON_VERSION="3.12" \
-  VALKEY_VERSION="8.0.1" \
-  POSTGRES_VERSION="15.5" \
+  PYTHON_VERSION="3.13" \
+  VALKEY_VERSION="8.1.0" \
+  POSTGRES_VERSION="15.12" \
   GOSU_VERSION="1.17"
 
 RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt
index ca19f67a2..9f542ed42 100644
--- a/rootfs/dev_requirements.txt
+++ b/rootfs/dev_requirements.txt
@@ -1,10 +1,10 @@
 # test module
 # test
 # Run "make test-unit" for the % of code exercised during tests
-coverage==7.6.1
+coverage==7.8.0
 
 # Run "make test-style" to check python syntax and style
-flake8==7.1.1
+flake8==7.2.0
 
 # mock out python-requests, mostly k8s
 requests-mock==1.12.1
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 8e1e8249a..b5c07f8ca 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -2,29 +2,29 @@
 inotify==0.2.10
 backoff==2.2.1
 django==4.2.20
-channels==4.1.0
-aiohttp==3.11.0
-django-cors-headers==4.6.0
+channels==4.2.2
+aiohttp==3.11.16
+django-cors-headers==4.7.0
 django-guardian==2.4.0
-djangorestframework==3.15.2
+djangorestframework==3.16.0
 docker==7.1.0
 gunicorn==23.0.0
-uvicorn[standard]==0.32.0
+uvicorn[standard]==0.34.0
 asgiref==3.8.1
 idna==3.10
 jsonschema==4.23.0
 morph==0.1.5
 packaging==24.2
 pyasn1==0.6.1
-psycopg[binary]==3.2.3
-pyOpenSSL==24.2.1
+psycopg[binary]==3.2.6
+pyOpenSSL==25.0.0
 ndg-httpsclient==0.5.1
-pytz==2024.2
+pytz==2025.2
 requests==2.32.3
 requests-toolbelt==1.0.0
-celery==5.4.0
-hiredis==3.0.0
+celery==5.5.0
+hiredis==3.1.0
 django_redis==5.4.0
 dj-database-url==2.3.0
-social-auth-app-django==5.4.2
-kubernetes==31.0.0
+social-auth-app-django==5.4.3
+kubernetes==32.0.1

From 9b75d4e6c70b17d02139edddb8017e09cff9f282 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 3 Apr 2025 13:12:20 +0800
Subject: [PATCH 124/200] chore(woodpecker): add cron event

---
 .woodpecker/build-linux.yml  |  1 +
 .woodpecker/chart.yaml       |  1 +
 .woodpecker/manifest.yml     |  2 ++
 .woodpecker/test-linux.yml   |  1 +
 rootfs/scheduler/__init__.py | 20 +++++++++-----------
 5 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml
index e9d18af8d..802f8754d 100644
--- a/.woodpecker/build-linux.yml
+++ b/.woodpecker/build-linux.yml
@@ -29,6 +29,7 @@ steps:
     event:
     - push
     - tag
+    - cron
 
 depends_on:
 - test-linux
\ No newline at end of file
diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml
index 40397e008..cfcd3ecbd 100644
--- a/.woodpecker/chart.yaml
+++ b/.woodpecker/chart.yaml
@@ -27,6 +27,7 @@ steps:
     event:
     - push
     - tag
+    - cron
 
 depends_on:
 - manifest
diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml
index d1f448393..88ded6677 100644
--- a/.woodpecker/manifest.yml
+++ b/.woodpecker/manifest.yml
@@ -15,6 +15,7 @@ steps:
     event:
     - tag
     - push
+    - cron
 
 - name: publish-manifest
   image: bash
@@ -36,6 +37,7 @@ steps:
     event:
     - tag
     - push
+    - cron
 
 depends_on:
 - build-linux
diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml
index ded271221..f22a5f0fe 100644
--- a/.woodpecker/test-linux.yml
+++ b/.woodpecker/test-linux.yml
@@ -23,3 +23,4 @@ steps:
     event:
     - push
     - tag
+    - cron
diff --git a/rootfs/scheduler/__init__.py b/rootfs/scheduler/__init__.py
index 6e5200e28..82aa8dadf 100644
--- a/rootfs/scheduler/__init__.py
+++ b/rootfs/scheduler/__init__.py
@@ -14,7 +14,6 @@
 
 logger = logging.getLogger(__name__)
 session = None
-resource_mapping = OrderedDict()
 
 
 def get_k8s_session(k8s_api_verify_tls):
@@ -40,9 +39,9 @@ class KubeHTTPClient(object):
     api_prefix = 'api'
     # ISO-8601 which is used by kubernetes
     DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+    resource_mapping = OrderedDict()
 
     def __init__(self, url, k8s_api_verify_tls=True):
-        global resource_mapping
         self.url = url
         self.k8s_api_verify_tls = k8s_api_verify_tls
         self.session = get_k8s_session(self.k8s_api_verify_tls)
@@ -53,33 +52,32 @@ def __init__(self, url, k8s_api_verify_tls=True):
             name = str(res.__name__).lower()  # singular
             component = name + 's'  # make plural
             # check if component has already been processed
-            if component in resource_mapping:
+            if component in self.resource_mapping:
                 continue
 
             # get past recursion problems in case of self reference
-            resource_mapping[component] = ''
-            resource_mapping[component] = res(self.url, self.k8s_api_verify_tls)
+            self.resource_mapping[component] = ''
+            self.resource_mapping[component] = res(self.url, self.k8s_api_verify_tls)
             # map singular Resource name to the plural one
-            resource_mapping[name] = component
+            self.resource_mapping[name] = component
             if res.short_name is not None:
                 # map short name to long name so a resource can be named rs
                 # but have the main object live at replicasets
-                resource_mapping[str(res.short_name).lower()] = component
+                self.resource_mapping[str(res.short_name).lower()] = component
 
     def api(self, tmpl, *args):
         """Return a fully-qualified Kubernetes API URL from a string template with args."""
         return "/{}/{}".format(self.api_prefix, self.api_version) + tmpl.format(*args)
 
     def __getattr__(self, name):
-        global resource_mapping
-        if name in resource_mapping:
+        if name in self.resource_mapping:
             # resolve to final name if needed
-            component = resource_mapping[name]
+            component = self.resource_mapping[name]
             if type(component) is not str:
                 # already a component object
                 return component
 
-            return resource_mapping[component]
+            return self.resource_mapping[component]
 
         return object.__getattribute__(self, name)
 

From 478cd6fdbb0e221e85e8e6b40641e54f7ce1a00f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 14 Apr 2025 17:29:21 +0800
Subject: [PATCH 125/200] fix(migrate): group migrate error

---
 rootfs/api/migrations/0021_migration_dryccfile.py | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/rootfs/api/migrations/0021_migration_dryccfile.py b/rootfs/api/migrations/0021_migration_dryccfile.py
index fc5dad40f..3bd0140ba 100644
--- a/rootfs/api/migrations/0021_migration_dryccfile.py
+++ b/rootfs/api/migrations/0021_migration_dryccfile.py
@@ -9,28 +9,16 @@ def migration_dryccfile(apps, schema_editor):
         build = release.build
         if not build or not build.dryccfile or "pipeline" in build.dryccfile:
             continue
-        config, envs = {}, {}
-        if release.config:
-            for value in release.config.values:
-                if "ptype" in value:
-                    if value["ptype"] not in envs:
-                        envs[value["ptype"]] = {}
-                    envs[value["ptype"]][value["name"]] = value["value"]
-                elif "group" in value:
-                    if value["group"] not in envs:
-                        config[value["group"]] = {}
-                    config[value["group"]][value["name"]] = value["value"]
         pipeline = {}
         for key, value in build.dryccfile.get('deploy', {}).items():
             pipeline.update({
                 "%s.yaml" % key: {
                     "kind": "pipeline",
                     "ptype": key,
-                    "env": envs.get(key, {}),
                     "deploy": value,
                 }
             })
-        build.dryccfile = {"config": config, "pipeline": pipeline}
+        build.dryccfile = {"pipeline": pipeline}
         build.save()
 
 

From eda22085cdbe69696c2633b151f2367af7d6afb7 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 18 Apr 2025 11:09:33 +0800
Subject: [PATCH 126/200] fix(monitor): memory usage error

---
 rootfs/api/monitor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index e09cc054e..2221d9e61 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -28,7 +28,7 @@
 
 
 query_memory_usage_promql_tpl = """
-sum (rate (container_memory_working_set_bytes{pod=~"^%s-.*$",namespace="%s"}[%s]))
+sum (avg_over_time (container_memory_working_set_bytes{pod=~"^%s-.*$",namespace="%s"}[%s]))
 by (pod)
 """
 

From 3d5b1f062133a12221f462842e9d0759004c70f0 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 30 Apr 2025 09:41:30 +0800
Subject: [PATCH 127/200] chore(filer): add bump filer version script

---
 .woodpecker/chart.yaml        | 1 +
 _scripts/util/update_filer.sh | 6 ++++++
 2 files changed, 7 insertions(+)
 create mode 100755 _scripts/util/update_filer.sh

diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml
index cfcd3ecbd..63eb318c5 100644
--- a/.woodpecker/chart.yaml
+++ b/.woodpecker/chart.yaml
@@ -10,6 +10,7 @@ steps:
   - export IMAGE_TAG=$([ ! -z $CI_COMMIT_TAG ] && echo \"$VERSION\" || echo \"canary\")
   - export APP_VERSION=$([ -z $CI_COMMIT_TAG ] && echo $CI_COMMIT_SHA || echo $VERSION)
   - export CHART_VERSION=$([ -z $CI_COMMIT_TAG ] && echo 1.0.0 || echo $VERSION)
+  - _scripts/util/update_filer.sh
   - sed -i "s/imageTag:\ \"canary\"/imageTag:\ $IMAGE_TAG/g" charts/$${CI_REPO_NAME}/values.yaml
   - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION
   - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin
diff --git a/_scripts/util/update_filer.sh b/_scripts/util/update_filer.sh
new file mode 100755
index 000000000..885b28a58
--- /dev/null
+++ b/_scripts/util/update_filer.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [ -n "${CI_COMMIT_TAG}" ]; then
+    version=$(curl -Ls https://github.com/drycc/filer/releases|grep /drycc/filer/releases/tag/ | sed -E 's/.*\/drycc\/filer\/releases\/tag\/(v[0-9\.]{1,}(-rc.[0-9]{1,})?)".*/\1/g' | head -1)
+    sed -i "s|registry.drycc.cc/drycc/filer:canary|registry.drycc.cc/drycc/filer:${version#v}|g" charts/controller/values.yaml
+fi
\ No newline at end of file

From 63efe5afb0ecfbadbc65ea829044c91b9cda8fe2 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 7 May 2025 09:01:16 +0800
Subject: [PATCH 128/200] chore(charts): remove rbac

---
 charts/controller/templates/controller-clusterrole.yaml       | 3 ---
 .../controller/templates/controller-clusterrolebinding.yaml   | 2 --
 charts/controller/values.yaml                                 | 4 ----
 3 files changed, 9 deletions(-)

diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index 503e0c61c..096da2ecc 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -1,4 +1,3 @@
-{{- if (.Values.global.rbac) -}}
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -88,5 +87,3 @@ rules:
 - apiGroups: ["batch"]
   resources: ["jobs"]
   verbs: ["create"]
-{{- end -}}
-
diff --git a/charts/controller/templates/controller-clusterrolebinding.yaml b/charts/controller/templates/controller-clusterrolebinding.yaml
index bf6517e94..23ee8d184 100644
--- a/charts/controller/templates/controller-clusterrolebinding.yaml
+++ b/charts/controller/templates/controller-clusterrolebinding.yaml
@@ -1,4 +1,3 @@
-{{- if (.Values.global.rbac) -}}
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -14,4 +13,3 @@ subjects:
 - kind: ServiceAccount
   name: drycc-controller
   namespace: {{ .Release.Namespace }}
-{{- end -}}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 795f7239f..6c19f2268 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -162,12 +162,8 @@ global:
   # - on-cluster: Run registry within the Kubernetes cluster
   # - off-cluster: Use registry outside the Kubernetes cluster (example: drycc registry,self-hosted)
   registryLocation: "on-cluster"
-  # The host port to which registry proxy binds to
-  registryProxyPort: 5555
   # Prefix for the imagepull secret created when using private registry
   registrySecretPrefix: "private-registry"
-  # Role-Based Access Control for Kubernetes >= 1.5
-  rbac: false
   # GatewayClass is cluster-scoped resource defined by the infrastructure provider.
   gatewayClass: ""
   # Please check `kubernetes.io/ingress.class`

From 10e6854c92f10253242282e5e8336784312a82a3 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 7 May 2025 18:01:39 +0800
Subject: [PATCH 129/200] chore(charts): use condition

---
 charts/controller/templates/_helpers.tpl      | 12 ++--
 .../templates/controller-clusterrole.yaml     |  9 ---
 .../templates/controller-secret-creds.yaml    |  5 ++
 charts/controller/values.yaml                 | 56 +++++++------------
 rootfs/api/models/app.py                      |  8 +--
 rootfs/api/tests/test_build.py                | 18 +++++-
 rootfs/api/tests/test_release.py              | 10 ++--
 rootfs/api/views.py                           |  5 ++
 rootfs/scheduler/mock.py                      | 10 ++--
 9 files changed, 64 insertions(+), 69 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 9a9774d8f..7fe40b0a7 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -11,9 +11,9 @@ env:
 - name: "K8S_API_VERIFY_TLS"
   value: "{{ .Values.k8sApiVerifyTls }}"
 - name: "DRYCC_REGISTRY_LOCATION"
-  value: "{{ .Values.global.registryLocation }}"
+  value: {{ ternary "on-cluster" "off-cluster" .Values.registry.enabled }}
 - name: "DRYCC_REGISTRY_SECRET_PREFIX"
-  value: "{{ .Values.global.registrySecretPrefix }}"
+  value: "{{ .Values.registrySecretPrefix }}"
 - name: "IMAGE_PULL_POLICY"
   value: "{{ .Values.appImagePullPolicy }}"
 - name: "DRYCC_FILER_IMAGE"
@@ -63,7 +63,7 @@ env:
     secretKeyRef:
       name: controller-creds
       key: valkey-url
-{{- else if eq .Values.global.valkeyLocation "on-cluster"  }}
+{{- else if .Values.valkey.enabled }}
 - name: VALKEY_PASSWORD
   valueFrom:
     secretKeyRef:
@@ -85,7 +85,7 @@ env:
     secretKeyRef:
       name: controller-creds
       key: database-url
-{{- else if eq .Values.global.databaseLocation "on-cluster"  }}
+{{- else if .Values.database.enabled }}
 - name: DRYCC_PG_USER
   valueFrom:
     secretKeyRef:
@@ -119,7 +119,7 @@ env:
     secretKeyRef:
       name: controller-creds
       key: prometheus-url
-{{- else if eq .Values.global.prometheusLocation "on-cluster" }}
+{{- else if .Values.prometheus.enabled }}
 - name: "DRYCC_PROMETHEUS_USERNAME"
   valueFrom:
     secretKeyRef:
@@ -133,7 +133,7 @@ env:
 - name: "DRYCC_PROMETHEUS_URL"
   value: "http://$(DRYCC_PROMETHEUS_USERNAME):$(DRYCC_PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090"
 {{- end }}
-{{- if eq .Values.global.passportLocation "on-cluster"}}
+{{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
 {{- if .Values.global.certManagerEnabled }}
   value: https://drycc-passport.{{ .Values.global.platformDomain }}
diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index 096da2ecc..611129a13 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -54,18 +54,9 @@ rules:
 - apiGroups: ["extensions", "autoscaling"]
   resources: ["horizontalpodautoscalers"]
   verbs: ["get", "list", "create", "update", "delete"]
-- apiGroups: ["extensions"]
-  resources: ["ingresses"]
-  verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: ["cert-manager.io"]
   resources: ["certificates", "certificaterequest", "issuers"]
   verbs: ["get", "list", "watch", "create", "update", "delete"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses"]
-  verbs: ["get", "list", "create", "delete", "deletecollection", "patch", "update"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingressclasses"]
-  verbs: ["get"]
 - apiGroups: ["apps"]
   resources: ["replicasets"]
   verbs: ["get", "list", "watch", "delete"]
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index 79684193f..3b98a9f2f 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -26,5 +26,10 @@ data:
   {{- if (.Values.passportSecret) }}
   passport-secret: {{ .Values.passportSecret | b64enc }}
   {{- end }}
+  {{- if (.Values.registryHost) }}
+  registry-host: {{ .Values.registryHost | b64enc }}
+  registry-username: {{ .Values.registryUsername | b64enc }}
+  registry-password: {{ .Values.registryPassword | b64enc }}
+  {{- end }}
   django-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "django-secret-key" "defaultValue" (randAscii 64) "context" $)) }}
   deploy-hook-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "deploy-hook-secret-key" "defaultValue" (randAscii 64) "context" $)) }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 6c19f2268..ff28fc6d2 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -35,7 +35,7 @@ appImagePullPolicy: "Always"
 filerImage: "registry.drycc.cc/drycc/filer:canary"
 filerImagePullPolicy: "Always"
 # Set storageClassName. It is used for application.
-appStorageClass: "{{ .Values.storage.csi.storageClassName }}"
+appStorageClass: "longhorn"
 # Set runtimeClassName. It is used for application.
 appRuntimeClass: ""
 # Set appDNSPolicy. It is used for application.
@@ -63,10 +63,14 @@ databaseReplicaUrl: ""
 passportUrl: ""
 passportKey: ""
 passportSecret: ""
+# prometheusUrl is will no longer use the built-in prometheus component
+prometheusUrl: ""
 # Workflow-manager Configuration Options
 workflowManagerUrl: ""
 workflowManagerAccessKey: ""
 workflowManagerSecretKey: ""
+# Prefix for the imagepull secret created when using private registry
+registrySecretPrefix: "private-registry"
 
 # limit specs, plans config
 config:
@@ -133,41 +137,26 @@ mutate:
     extraMatchLabels:
       component: "drycc-controller-mutate"
 
-storage:
-  csi:
-    storageClassName: drycc-storage
+valkey:
+  enabled: true
+
+database:
+  enabled: true
+
+registry:
+  enabled: true
+
+passport:
+  enabled: true
+
+prometheus:
+  enabled: true
 
 global:
   # Admin email, used for each component to send email to administrator
   email: "drycc@drycc.cc"
-  # Set the storage backend
-  #
-  # Valid values are:
-  # - s3: Store persistent data in AWS S3 (configure in S3 section)
-  # - azure: Store persistent data in Azure's object storage
-  # - gcs: Store persistent data in Google Cloud Storage
-  # - minio: Store persistent data on in-cluster Minio server
-  storage: minio
-  # Set the location of Workflow's PostgreSQL database
-  #
-  # Valid values are:
-  # - on-cluster: Run PostgreSQL within the Kubernetes cluster (credentials are generated
-  #   automatically; backups are sent to object storage
-  #   configured above)
-  # - off-cluster: Run PostgreSQL outside the Kubernetes cluster (configure in database section)
-  databaseLocation: "on-cluster"
-  # Set the location of Workflow's Registry
-  #
-  # Valid values are:
-  # - on-cluster: Run registry within the Kubernetes cluster
-  # - off-cluster: Use registry outside the Kubernetes cluster (example: drycc registry,self-hosted)
-  registryLocation: "on-cluster"
-  # Prefix for the imagepull secret created when using private registry
-  registrySecretPrefix: "private-registry"
   # GatewayClass is cluster-scoped resource defined by the infrastructure provider.
   gatewayClass: ""
-  # Please check `kubernetes.io/ingress.class`
-  ingressClass: ""
   # A domain name consists of one or more parts.
   # Periods (.) are used to separate these parts.
   # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-).
@@ -179,10 +168,3 @@ global:
   platformDomain: ""
   # Whether cert_manager is enabled to automatically generate controller certificates
   certManagerEnabled: true
-  passportLocation: "on-cluster"
-  # Set the location of Workflow's prometheus cluster
-  #
-  # Valid values are:
-  # - on-cluster: Run prometheus within the Kubernetes cluster
-  # - off-cluster: prometheus is running outside of the cluster and credentials and connection information will be provided.
-  prometheusLocation: "on-cluster"
\ No newline at end of file
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index c3464127d..1c28aef9f 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1149,12 +1149,12 @@ def _get_private_registry_config(self, ptype, image, registry=None):
             password = registry.get('password')
         elif settings.REGISTRY_LOCATION == 'off-cluster':
             secret = self.scheduler().secret.get(
-                settings.WORKFLOW_NAMESPACE, 'registry-secret').json()
-            username = secret['data']['username']
-            password = secret['data']['password']
-            hostname = secret['data']['hostname']
+                settings.WORKFLOW_NAMESPACE, 'controller-creds').json()
+            hostname = secret['data']['registry-host']
             if hostname == '':
                 hostname = docker_auth.INDEX_URL
+            username = secret['data']['registry-username']
+            password = secret['data']['registry-password']
             name = name + '-' + settings.REGISTRY_LOCATION
         else:
             return None, None, None
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index fbd7c8625..6e30e172a 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -510,13 +510,13 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
-    def test_build_image_in_registry(self, mock_requests):
+    def test_build_image_in_registry_ok(self, mock_requests):
         """When the image is already in the drycc registry no pull/tag/push happens"""
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
         url = f"/v2/apps/{app_id}/build"
-        image = '127.0.0.1:5000/autotest/example'
+        image = 'registry.drycc.cc:5000/autotest/example'
         body = {'image': image, 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -527,7 +527,7 @@ def test_build_image_in_registry(self, mock_requests):
 
         # post an image as a build using registry hostname + port
         url = f"/v2/apps/{app_id}/build"
-        image = '127.0.0.1:5000/autotest/example'
+        image = 'registry.drycc.cc:5000/autotest/example'
         body = {'image': image, 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -536,6 +536,18 @@ def test_build_image_in_registry(self, mock_requests):
         release = build.app.release_set.latest()
         self.assertEqual(release.get_deploy_image(PTYPE_WEB), image)
 
+    def test_build_image_in_registry_err(self, mock_requests):
+        """When the image is already in the drycc registry no pull/tag/push happens"""
+        app_id = self.create_app()
+
+        # post an image as a build using registry hostname
+        url = f"/v2/apps/{app_id}/build"
+        for host in ['127.0.0.1', 'localhost', 'localhost:5000', '127.0.0.1:5000']:
+            image = f'{host}/autotest/example'
+            body = {'image': image, 'stack': 'container'}
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 400, response.data)
+
     def test_build_image_in_registry_with_auth(self, mock_requests):
         """add authentication to the build"""
         app_id = self.create_app()
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 825b18918..8f405702f 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -138,7 +138,7 @@ def test_get_image(self, mock_requests):
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/build"
         body = {
-            'image': '127.0.0.1:5555/autotest/example:git-fadf1231',
+            'image': 'registry.drycc.cc:5555/autotest/example:git-fadf1231',
             'stack': 'heroku-18',
             'sha': 'a'*40,
             'dryccfile': {
@@ -206,15 +206,15 @@ def test_get_image(self, mock_requests):
                         "deploy": {
                             "command": ["bash", "-c"],
                             "args": ["bundle exec puma -C config/puma.rb"],
-                            "image": "127.0.0.1:7070/myapp/web:git-123fsa1",
+                            "image": "registry.drycc.cc:7070/myapp/web:git-123fsa1",
                         },
                     },
                 },
             },
         }
-        default_image = '127.0.0.1:5555/autotest/example:git-fadf1231'
-        worker_image = "127.0.0.1:5555/autotest/example:git-fadf1231-worker"
-        worker_4_image = "127.0.0.1:7070/myapp/web:git-123fsa1"
+        default_image = 'registry.drycc.cc:5555/autotest/example:git-fadf1231'
+        worker_image = "registry.drycc.cc:5555/autotest/example:git-fadf1231-worker"
+        worker_4_image = "registry.drycc.cc:7070/myapp/web:git-123fsa1"
 
         with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
             mock_kube.return_value = ['up', 'down']
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 734489102..f72f3c755 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -44,6 +44,7 @@
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
+is_loopback = re.compile(r'^(localhost|127\.0\.0\.1)(:\d+)?/').match
 oauth_cache_manager = OauthCacheManager()
 NAMESPACE = getattr(settings, setting_name('URL_NAMESPACE'), None) or 'social'
 
@@ -352,6 +353,10 @@ class BuildViewSet(ReleasableViewSet):
     serializer_class = serializers.BuildSerializer
 
     def post_save(self, build):
+        for ptype in build.ptypes:
+            image = build.get_image(ptype)
+            if is_loopback(image):
+                raise DryccException("image must not use the loopback address")
         build.create_release(self.request.user)
         super(BuildViewSet, self).post_save(build)
 
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index 12e28a685..d19b70854 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -1093,14 +1093,14 @@ def __init__(self, url, k8s_api_verify_tls=True):
             self.secret.create('drycc', 'objectstorage-keyfile', secrets)
 
         try:
-            self.secret.get('drycc', 'registry-secret')
+            self.secret.get('drycc', 'controller-creds')
         except KubeHTTPException:
             secrets = {
-                'username': 'test',
-                'password': 'test',
-                'hostname': ''
+                'registry-host': '',
+                'registry-username': 'test',
+                'registry-password': 'test',
             }
-            self.secret.create('drycc', 'registry-secret', secrets)
+            self.secret.create('drycc', 'controller-creds', secrets)
 
         try:
             self.ns.get('duplicate')

From 17259dbb4f9ea3ee0183b00e806340f27e6fc87f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 8 May 2025 15:45:20 +0800
Subject: [PATCH 130/200] fix(oidc): got an unexpected keyword argument timeout

---
 rootfs/api/backend.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/rootfs/api/backend.py b/rootfs/api/backend.py
index bfa91e871..f629f15a6 100644
--- a/rootfs/api/backend.py
+++ b/rootfs/api/backend.py
@@ -33,14 +33,12 @@ class DryccOIDC(OpenIdConnectAuth):
     ]
 
     def __init__(self, *args, **kwargs):
-        self.timeout = 3  # request timeout
         super().__init__(*args, **kwargs)
 
     @social_cache(ttl=86400)
     def oidc_config(self):
         return self.get_json(
-            self.OIDC_ENDPOINT + '/.well-known/openid-configuration/',
-            timeout=self.timeout
+            self.OIDC_ENDPOINT + '/.well-known/openid-configuration/'
         )
 
     def get_user_data(self, access_token):
@@ -51,7 +49,6 @@ def get_user_data(self, access_token):
             headers={
                 'authorization': 'Bearer ' + access_token
             },
-            timeout=self.timeout,
         )
         return {
             'id': response.get('id'),
@@ -73,7 +70,6 @@ def refresh_token(self, refresh_token):
                 'client_id': settings.SOCIAL_AUTH_DRYCC_KEY,
                 'refresh_token': refresh_token,
             },
-            timeout=self.timeout,
         )
 
 

From ffe45f0a6bbd8de2a4ce912f19cff4d00dc5059c Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 12 May 2025 09:09:52 +0800
Subject: [PATCH 131/200] chore(charts): change resources format

---
 charts/controller/templates/_helpers.tpl      | 14 -----------
 .../templates/controller-api-deployment.yaml  |  5 +++-
 .../controller-celery-deloyment.yaml          |  5 +++-
 .../controller-mutate-deloyment.yaml          |  5 +++-
 charts/controller/values.yaml                 | 23 +++++++++++++++++--
 5 files changed, 33 insertions(+), 19 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 7fe40b0a7..236a7677a 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -180,20 +180,6 @@ env:
   value: api.routers.DefaultReplicaRouter
 {{- end }}
 
-{{/* Generate controller deployment limits */}}
-{{- define "controller.limits" -}}
-{{- if or (.Values.limitsCpu) (.Values.limitsMemory) }}
-resources:
-  limits:
-{{- if (.Values.limitsCpu) }}
-    cpu: {{.Values.limitsCpu}}
-{{- end }}
-{{- if (.Values.limitsMemory) }}
-    memory: {{.Values.limitsMemory}}
-{{- end }}
-{{- end }}
-{{- end }}
-
 {{/* Generate controller config default limit specs */}}
 {{ define "controller.config.defaultLimitSpecs" }}
 - model: api.limitspec
diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/controller-api-deployment.yaml
index d5b1971f4..e92777e5b 100644
--- a/charts/controller/templates/controller-api-deployment.yaml
+++ b/charts/controller/templates/controller-api-deployment.yaml
@@ -70,7 +70,10 @@ spec:
         ports:
           - containerPort: 8000
             name: http
-        {{- include "controller.limits" . | indent 8 }}
+        {{- with index .Values "api" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         {{- include "controller.envs" . | indent 8 }}
         volumeMounts:
         - name: controller-config
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/controller-celery-deloyment.yaml
index 10972d934..c11f26b61 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/controller-celery-deloyment.yaml
@@ -51,7 +51,10 @@ spec:
           - -c
           - celery --app api worker --queues controller.low,controller.middle,controller.high --autoscale=32,1 --loglevel=WARNING
         {{- end }}
-        {{- include "controller.limits" $ | indent 8 }}
+        {{- with index .Values "celery" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         {{- include "controller.envs" $ | indent 8 }}
         volumeMounts:
         - name: controller-config
diff --git a/charts/controller/templates/controller-mutate-deloyment.yaml b/charts/controller/templates/controller-mutate-deloyment.yaml
index def3c3ce0..4f20a0e74 100644
--- a/charts/controller/templates/controller-mutate-deloyment.yaml
+++ b/charts/controller/templates/controller-mutate-deloyment.yaml
@@ -59,7 +59,10 @@ spec:
         volumeMounts:
         - name: controller-mutate-cert
           mountPath: /etc/controller/mutate/cert
-        {{- include "controller.limits" . | indent 8 }}
+        {{- with index .Values "mutate" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         {{- include "controller.envs" . | indent 8 }}
       volumes:
       - name: controller-mutate-cert                                                                                                                                                                                               
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index ff28fc6d2..364392cc3 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -20,8 +20,6 @@ diagnosticMode:
 # See https://www.drycc.cc/managing-workflow/deploy-hooks/
 deployHookUrls: ""
 timezone: "UTC"
-# limitsCpu: "100m"
-# limitsMemory: "50Mi"
 # Possible values are:
 # enabled - allows for open registration
 # disabled - turns off open registration
@@ -93,6 +91,13 @@ environment:
   RESERVED_NAMES: "drycc, drycc-builder, drycc-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
 
 api:
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
   nodeAffinityPreset:
     key: "drycc.cc/node"
     type: "soft"
@@ -108,6 +113,13 @@ api:
       app: "drycc-controller-api"
 
 celery:
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
   nodeAffinityPreset:
     key: "drycc.cc/node"
     type: "soft"
@@ -123,6 +135,13 @@ celery:
       app: "drycc-controller-celery"
 
 mutate:
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
   nodeAffinityPreset:
     key: "drycc.cc/node"
     type: "soft"

From 877e35c2b2991c2ea515b58aabb6659add59c8fc Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 12 May 2025 09:10:45 +0800
Subject: [PATCH 132/200] chore(deps): bump django from 4.2.20 to 4.2.21 in
 /rootfs (#182)

Bumps [django](https://github.com/django/django) from 4.2.20 to 4.2.21.
- [Commits](https://github.com/django/django/compare/4.2.20...4.2.21)

---
updated-dependencies:
- dependency-name: django
  dependency-version: 4.2.21
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index b5c07f8ca..2d2bca86c 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.20
+django==4.2.21
 channels==4.2.2
 aiohttp==3.11.16
 django-cors-headers==4.7.0

From 80353fc09debecbee6ac56aa3b3248470c400605 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 13 May 2025 13:06:13 +0800
Subject: [PATCH 133/200] chore(charts): change prometheus url

---
 charts/controller/templates/_helpers.tpl | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 236a7677a..285a6a1ef 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -120,18 +120,8 @@ env:
       name: controller-creds
       key: prometheus-url
 {{- else if .Values.prometheus.enabled }}
-- name: "DRYCC_PROMETHEUS_USERNAME"
-  valueFrom:
-    secretKeyRef:
-      name: prometheus-creds
-      key: username
-- name: "DRYCC_PROMETHEUS_PASSWORD"
-  valueFrom:
-    secretKeyRef:
-      name: prometheus-creds
-      key: password
 - name: "DRYCC_PROMETHEUS_URL"
-  value: "http://$(DRYCC_PROMETHEUS_USERNAME):$(DRYCC_PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090"
+  value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481/select/0/prometheus"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"

From 01d6d6c85269dc0509e5173d262739f682fcd789 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 13 May 2025 13:42:54 +0800
Subject: [PATCH 134/200] chore(controller): change app storage class name

---
 charts/controller/templates/_helpers.tpl | 5 ++---
 charts/controller/values.yaml            | 7 +++----
 rootfs/api/models/gateway.py             | 2 +-
 rootfs/api/settings/production.py        | 5 ++---
 4 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 285a6a1ef..d4d87618c 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -5,9 +5,6 @@ env:
   value: {{ .Chart.AppVersion }}
 - name: REGISTRATION_MODE
   value: {{ .Values.registrationMode }}
-# Environmental variable value for $GATEWAY_CLASS
-- name: "DRYCC_GATEWAY_CLASS"
-  value: "{{ .Values.global.gatewayClass }}"
 - name: "K8S_API_VERIFY_TLS"
   value: "{{ .Values.k8sApiVerifyTls }}"
 - name: "DRYCC_REGISTRY_LOCATION"
@@ -22,6 +19,8 @@ env:
   value: "{{ (tpl .Values.filerImagePullPolicy .) }}"
 - name: "KUBERNETES_CLUSTER_DOMAIN"
   value: "{{ .Values.global.clusterDomain }}"
+- name: "DRYCC_APP_GATEWAY_CLASS"
+  value: "{{ .Values.appGatewayClass }}"
 {{- if (.Values.appStorageClass) }}
 - name: "DRYCC_APP_STORAGE_CLASS"
   value: "{{ (tpl .Values.appStorageClass .) }}"
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 364392cc3..7756dc449 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -32,7 +32,9 @@ appImagePullPolicy: "Always"
 # Filer settings image and image pull policy
 filerImage: "registry.drycc.cc/drycc/filer:canary"
 filerImagePullPolicy: "Always"
-# Set storageClassName. It is used for application.
+# Set app gateway, It is used for application gateway.
+appGatewayClass: ""
+# Set storageClassName, It is used for application mount.
 appStorageClass: "longhorn"
 # Set runtimeClassName. It is used for application.
 appRuntimeClass: ""
@@ -69,7 +71,6 @@ workflowManagerAccessKey: ""
 workflowManagerSecretKey: ""
 # Prefix for the imagepull secret created when using private registry
 registrySecretPrefix: "private-registry"
-
 # limit specs, plans config
 config:
   metrics: ""
@@ -174,8 +175,6 @@ prometheus:
 global:
   # Admin email, used for each component to send email to administrator
   email: "drycc@drycc.cc"
-  # GatewayClass is cluster-scoped resource defined by the infrastructure provider.
-  gatewayClass: ""
   # A domain name consists of one or more parts.
   # Periods (.) are used to separate these parts.
   # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-).
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 6aa15cccb..55c42a638 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -93,7 +93,7 @@ def listeners(self):
         return listeners
 
     def refresh_to_k8s(self):
-        kwargs = {"listeners": self.listeners, "gateway_class": settings.GATEWAY_CLASS}
+        kwargs = {"listeners": self.listeners, "gateway_class": settings.DRYCC_APP_GATEWAY_CLASS}
         if self.app.tls_set.latest().certs_auto_enabled:
             kwargs["annotations"] = {"cert-manager.io/issuer": self.app.id}
         try:
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index a05c59142..8d6021c97 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -319,9 +319,6 @@
 else:
     MUTATE_KEY = None
 
-# gateway class name
-GATEWAY_CLASS = os.environ.get('DRYCC_GATEWAY_CLASS', '')
-
 IMAGE_PULL_POLICY = os.environ.get('IMAGE_PULL_POLICY', "IfNotPresent")
 
 # apply task size
@@ -357,6 +354,8 @@
 
 DRYCC_DEPLOY_HOOK_SECRET_KEY = os.environ.get('DRYCC_DEPLOY_HOOK_SECRET_KEY', None)
 
+DRYCC_APP_GATEWAY_CLASS = os.environ.get('DRYCC_APP_GATEWAY_CLASS', "")
+
 DRYCC_APP_STORAGE_CLASS = os.environ.get('DRYCC_APP_STORAGE_CLASS', "")
 
 DRYCC_APP_DNS_POLICY = os.environ.get('DRYCC_APP_DNS_POLICY', "")

From 1e6035974ee52b1fcce149e3ebad25bd5d50a869 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 23 May 2025 11:56:55 +0800
Subject: [PATCH 135/200] feat(controller): add drycc annotation to Kubernetes

---
 rootfs/api/models/app.py         | 85 ++++++++++++++++++--------------
 rootfs/api/models/base.py        | 15 +++---
 rootfs/api/models/certificate.py | 10 ++--
 rootfs/api/models/config.py      |  2 +-
 rootfs/api/models/gateway.py     | 28 +++++------
 rootfs/api/models/release.py     | 20 ++++----
 rootfs/api/models/resource.py    | 38 +++++++-------
 rootfs/api/models/service.py     |  8 +--
 rootfs/api/models/tls.py         | 26 +++++-----
 rootfs/api/models/volume.py      | 30 +++++------
 rootfs/api/tests/test_gateway.py |  8 +--
 rootfs/api/utils.py              |  8 +++
 rootfs/api/views.py              |  2 +-
 rootfs/scheduler/__init__.py     | 11 ++++-
 rootfs/scheduler/mock.py         |  4 +-
 15 files changed, 162 insertions(+), 133 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 1c28aef9f..7dd2fa176 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -133,7 +133,7 @@ def save(self, *args, **kwargs):
             self.release_set.latest()
         except Release.DoesNotExist:
             try:
-                if self.scheduler().ns.get(self.id).status_code == 200:
+                if self.scheduler.ns.get(self.id).status_code == 200:
                     # Namespace already exists
                     err = "{} already exists as a namespace in this kuberenetes setup".format(self.id)  # noqa
                     self.log(err, logging.INFO)
@@ -155,6 +155,17 @@ def lock(self):
     def ptypes(self):
         return list(self.structure.keys())
 
+    @property
+    def scheduler(self):
+        """
+        Override @Base.AuditedModel.scheduler;
+        since the app itself doesn't have an app object context,
+        directly reference using ID instead.
+        """
+        scheduler = super(App, self).scheduler
+        scheduler.metadata["annotations"]["drycc.cc/project_id"] = str(self.id)
+        return scheduler
+
     def check_ptypes(self, ptypes: set):
         """
         check available procfile types
@@ -199,10 +210,10 @@ def create(self, *args, **kwargs):  # noqa
         self.log('creating Namespace {} and services'.format(namespace), level=logging.DEBUG)
         # Create essential resources
         try:
-            self.scheduler().ns.get(namespace)
+            self.scheduler.ns.get(namespace)
         except KubeException:
             try:
-                self.scheduler().ns.create(namespace)
+                self.scheduler.ns.create(namespace)
             except KubeException as e:
                 raise ServiceUnavailable('Could not create the Namespace in Kubernetes') from e
         try:
@@ -220,15 +231,15 @@ def delete(self, *args, **kwargs):
         self.log("deleting environment")
         try:
             # check if namespace exists
-            self.scheduler().ns.get(self.id)
+            self.scheduler.ns.get(self.id)
 
             try:
-                self.scheduler().ns.delete(self.id)
+                self.scheduler.ns.delete(self.id)
 
                 # wait 30 seconds for termination
                 for _ in range(30):
                     try:
-                        self.scheduler().ns.get(self.id)
+                        self.scheduler.ns.get(self.id)
                     except KubeHTTPException as e:
                         # only break out on a 404
                         if e.response.status_code == 404:
@@ -254,7 +265,7 @@ def restart(self, **kwargs):  # noqa
             tasks = [
                 (
                     functools.partial(
-                        self.scheduler().deployment.restart,
+                        self.scheduler.deployment.restart,
                         self.id,
                         deployment
                     ),
@@ -293,7 +304,7 @@ def pipeline(self, release, ptypes, force_deploy=False):
                             envs=self._build_env_vars(release, run['ptype']),
                         )
                         state, labels = 'initializing', {'job-name': job_name}
-                        for count, state in enumerate(self.scheduler().pod.watch(
+                        for count, state in enumerate(self.scheduler.pod.watch(
                                 self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
                             self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
                         if state != 'down':
@@ -371,7 +382,7 @@ def clean(self, release=None, ptypes=None):
         labels = {'heritage': 'drycc'}
         if ptypes:
             labels["type__in"] = ptypes
-        resource_apis = [self.scheduler().deployments, self.scheduler().secret]
+        resource_apis = [self.scheduler.deployments, self.scheduler.secret]
         for api in resource_apis:
             resources = api.get(self.id, labels=labels).json()["items"]
             if resources is not None:
@@ -414,7 +425,7 @@ def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
         try:
             # create application config and build the pod manifest
             self.set_application_config(release, PTYPE_RUN)
-            self.scheduler().job.create(self.id, name, image, command, args, **kwargs)
+            self.scheduler.job.create(self.id, name, image, command, args, **kwargs)
         except Exception as e:
             err = '{} ({}): {}'.format(name, PTYPE_RUN, e)
             raise ServiceUnavailable(err) from e
@@ -431,7 +442,7 @@ def get_command_and_args(pod, container_name):
             return command, args
         result = []
         try:
-            pod = self.scheduler().pod.get(self.id, pod_name).json()
+            pod = self.scheduler.pod.get(self.id, pod_name).json()
             if pod["status"]['phase'] == 'Pending':
                 statuses = pod["spec"]["containers"]
             else:
@@ -462,9 +473,9 @@ def list_pods(self, *args, **kwargs):
             labels = self._scheduler_filter(**kwargs)
             # in case a singular pod is requested
             if 'name' in kwargs:
-                pods = [self.scheduler().pod.get(self.id, kwargs['name']).json()]
+                pods = [self.scheduler.pod.get(self.id, kwargs['name']).json()]
             else:
-                pods = self.scheduler().pod.get(self.id, labels=labels).json()['items']
+                pods = self.scheduler.pod.get(self.id, labels=labels).json()['items']
                 if not pods:
                     pods = []
             data = []
@@ -475,7 +486,7 @@ def list_pods(self, *args, **kwargs):
                 else:
                     started = str(
                         datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT))
-                state = str(self.scheduler().pod.state(p))
+                state = str(self.scheduler.pod.state(p))
                 if p['status']['phase'] != 'Pending':
                     ready = len([1 for s in p["status"]["containerStatuses"] if s['ready']])
                     restarts = sum([s['restartCount'] for s in p["status"]["containerStatuses"]])
@@ -508,9 +519,9 @@ def delete_pod(self, **kwargs):
         pod_name = kwargs.get('pod_name')
         try:
             # make sure the pod is manageed by drycc
-            pod = self.scheduler().pod.get(self.id, pod_name).json()
+            pod = self.scheduler.pod.get(self.id, pod_name).json()
             if pod['metadata']['labels'].get("heritage") == "drycc":
-                self.scheduler().pod.delete(self.id, pod_name)
+                self.scheduler.pod.delete(self.id, pod_name)
         except KubeHTTPException as e:
             # Sometimes k8s will manage to remove the pod from under us
             if e.response.status_code != 404:
@@ -519,7 +530,7 @@ def delete_pod(self, **kwargs):
     def describe_deployment(self, deployment_name):
         result = []
         try:
-            deployment = self.scheduler().deployment.get(self.id, deployment_name).json()
+            deployment = self.scheduler.deployment.get(self.id, deployment_name).json()
             for container in deployment["spec"]["template"]['spec']["containers"]:
                 limits = container.get("resources", {}).get("limits", {})
                 result.append({
@@ -545,9 +556,9 @@ def list_deployments(self, *args, **kwargs):
             labels = self._scheduler_filter(**kwargs)
             # in case a singular deployment is requested
             if 'name' in kwargs:
-                deployments = [self.scheduler().deployment.get(self.id, kwargs['name']).json()]
+                deployments = [self.scheduler.deployment.get(self.id, kwargs['name']).json()]
             else:
-                deployments = self.scheduler().deployment.get(self.id, labels=labels).json()['items']  # noqa
+                deployments = self.scheduler.deployment.get(self.id, labels=labels).json()['items']  # noqa
                 if not deployments:
                     deployments = []
             data = []
@@ -588,7 +599,7 @@ def list_events(self, ref_kind, ref_name, *args, **kwargs):
                 "regarding.name": ref_name
             }
             kwargs["fields"] = fields
-            events = self.scheduler().events.get(self.id, **kwargs).json()['items']  # noqa
+            events = self.scheduler.events.get(self.id, **kwargs).json()['items']  # noqa
             data = []
             for e in events:
                 item = {
@@ -622,16 +633,16 @@ def autoscale(self, proc_type, autoscale):
 
         try:
             # get the target for autoscaler, in this case Deployment
-            self.scheduler().hpa.get(self.id, name)
+            self.scheduler.hpa.get(self.id, name)
             if autoscale is None:
-                self.scheduler().hpa.delete(self.id, name)
+                self.scheduler.hpa.delete(self.id, name)
             else:
-                self.scheduler().hpa.update(
+                self.scheduler.hpa.update(
                     self.id, name, proc_type, target, **autoscale
                 )
         except KubeHTTPException as e:
             if e.response.status_code == 404:
-                self.scheduler().hpa.create(
+                self.scheduler.hpa.create(
                     self.id, name, proc_type, target, **autoscale
                 )
             else:
@@ -649,16 +660,16 @@ def image_pull_secret(self, namespace, ptype, registry, image):
         elif create:
             data = {'.dockerconfigjson': docker_config}
             try:
-                self.scheduler().secret.get(namespace, name)
+                self.scheduler.secret.get(namespace, name)
             except KubeHTTPException:
-                self.scheduler().secret.create(
+                self.scheduler.secret.create(
                     namespace,
                     name,
                     data,
                     secret_type='kubernetes.io/dockerconfigjson'
                 )
             else:
-                self.scheduler().secret.update(
+                self.scheduler.secret.update(
                     namespace,
                     name,
                     data,
@@ -674,7 +685,7 @@ def state_to_k8s(self):
             return
         ptypes = set()
         for ptype, scale in self.structure.items():
-            response = self.scheduler().deployment.get(
+            response = self.scheduler.deployment.get(
                 self.id, self._get_deployment_name(ptype),
                 ignore_exception=True)
             if response.status_code == 404 and scale > 0:
@@ -710,11 +721,11 @@ def set_application_config(self, release, ptype):
 
         secret_name = "{}-{}-{}-env".format(self.id, ptype, release.version_name)
         try:
-            self.scheduler().secret.get(self.id, secret_name)
+            self.scheduler.secret.get(self.id, secret_name)
         except KubeHTTPException:
-            self.scheduler().secret.create(self.id, secret_name, secrets_env, labels=labels)
+            self.scheduler.secret.create(self.id, secret_name, secrets_env, labels=labels)
         else:
-            self.scheduler().secret.update(self.id, secret_name, secrets_env, labels=labels)
+            self.scheduler.secret.update(self.id, secret_name, secrets_env, labels=labels)
 
     def to_measurements(self, timestamp: float):
         measurements = []
@@ -766,7 +777,7 @@ def _mount(self, user, volume, app_settings, structure=None):
                     volume for volume in volumes if scale_type in volume.path.keys()]
                 data = self._gather_app_settings(
                     release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
-                deployment = self.scheduler().deployment.get(
+                deployment = self.scheduler.deployment.get(
                     self.id, self._get_deployment_name(scale_type)).json()
                 spec_annotations = deployment['spec']['template']['metadata'].get(
                     'annotations', {})
@@ -774,7 +785,7 @@ def _mount(self, user, volume, app_settings, structure=None):
                 # gather volume proc types to be deployed
                 tasks.append((
                     functools.partial(
-                        self.scheduler().deployment.patch,
+                        self.scheduler.deployment.patch,
                         namespace=self.id,
                         name=self._get_deployment_name(scale_type),
                         image=release.get_deploy_image(scale_type),
@@ -808,7 +819,7 @@ def _deploy(self, deploys, ptypes, prev_release,
                 self.set_application_config(release, scale_type)
                 tasks.append((
                     functools.partial(
-                        self.scheduler().deploy,
+                        self.scheduler.deploy,
                         namespace=self.id,
                         name=self._get_deployment_name(scale_type),
                         image=release.get_deploy_image(scale_type),
@@ -906,7 +917,7 @@ def _scale_pods(self, scale_types, release, app_settings):
             # gather all proc types to be deployed
             tasks.append((
                 functools.partial(
-                    self.scheduler().scale,
+                    self.scheduler.scale,
                     namespace=self.id,
                     name=self._get_deployment_name(scale_type),
                     image=release.get_deploy_image(scale_type),
@@ -1064,7 +1075,7 @@ def _check_deployment_in_progress(self, deploys, force_deploy=False):
         for scale_type, kwargs in deploys.items():
             name = self._get_deployment_name(scale_type)
             # Is there an existing deployment in progress?
-            in_progress, deploy_okay = self.scheduler().deployment.in_progress(
+            in_progress, deploy_okay = self.scheduler.deployment.in_progress(
                 self.id, name, kwargs.get("deploy_timeout"), kwargs.get("deploy_batches"),
                 kwargs.get("replicas"), kwargs.get("tags")
             )
@@ -1148,7 +1159,7 @@ def _get_private_registry_config(self, ptype, image, registry=None):
             username = registry.get('username')
             password = registry.get('password')
         elif settings.REGISTRY_LOCATION == 'off-cluster':
-            secret = self.scheduler().secret.get(
+            secret = self.scheduler.secret.get(
                 settings.WORKFLOW_NAMESPACE, 'controller-creds').json()
             hostname = secret['data']['registry-host']
             if hostname == '':
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 7e66d6136..211f7ff68 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -1,7 +1,6 @@
 import uuid
 import string
 import random
-import importlib
 from datetime import timedelta
 from functools import partial
 from django.db import models
@@ -11,7 +10,7 @@
 from django.utils.translation import gettext_lazy as _
 
 
-from api.utils import validate_json
+from api.utils import validate_json, get_scheduler
 
 token_manager_oauth_schema = {
     "$schema": "http://json-schema.org/schema#",
@@ -48,10 +47,14 @@ class Meta:
         """Mark :class:`AuditedModel` as abstract."""
         abstract = True
 
-    @classmethod
-    def scheduler(cls):
-        mod = importlib.import_module(settings.SCHEDULER_MODULE)
-        return mod.SchedulerClient(settings.SCHEDULER_URL, settings.K8S_API_VERIFY_TLS)
+    @property
+    def scheduler(self):
+        annotations = {}
+        if hasattr(self, 'app'):
+            annotations["drycc.cc/project_id"] = str(self.app.id)
+        if hasattr(self, 'owner'):
+            annotations["drycc.cc/account_id"] = str(self.owner.id)
+        return get_scheduler(metadata={"annotations": annotations})
 
 
 class UuidAuditedModel(AuditedModel):
diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index b1f9cd5b1..43b862427 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -207,14 +207,14 @@ def attach_in_kubernetes(self, domain):
                 'tls.key': self.key
             }
 
-            secret = self.scheduler().secret.get(namespace, self.certname).json()['data']
+            secret = self.scheduler.secret.get(namespace, self.certname).json()['data']
         except KubeException:
-            self.scheduler().secret.create(namespace, self.certname, data)
+            self.scheduler.secret.create(namespace, self.certname, data)
         else:
             # update cert secret to the TLS Ingress format if required
             if secret != data:
                 try:
-                    self.scheduler().secret.update(namespace, self.certname, data)
+                    self.scheduler.secret.update(namespace, self.certname, data)
                 except KubeException as e:
                     msg = 'There was a problem updating the certificate secret ' \
                           '{} for {}'.format(self.certname, namespace)
@@ -232,8 +232,8 @@ def detach(self, *args, **kwargs):
         if len(self.domains) == 0:
             try:
                 # We raise an exception when a secret doesn't exist
-                self.scheduler().secret.get(namespace, self.certname)
-                self.scheduler().secret.delete(namespace, self.certname)
+                self.scheduler.secret.get(namespace, self.certname)
+                self.scheduler.secret.delete(namespace, self.certname)
             except KubeException as e:
                 raise ServiceUnavailable(
                     "Could not delete certificate secret {} for application {}".format(
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 4c7742f31..1f7ba80c6 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -236,7 +236,7 @@ def _update_tags(self, previous_config, replace_ptypes=[]):
                         '{} does not exist under {}'.format(ptype, 'tags'))
                 data.pop(ptype)
             else:
-                if not self.scheduler().node.get(labels=values).json()['items']:
+                if not self.scheduler.node.get(labels=values).json()['items']:
                     labels = ['{}={}'.format(key, value) for key, value in values.items()]
                     message = 'No nodes matched the provided labels: {}'.format(', '.join(labels))
                     # Find out if there are any other tags around
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 55c42a638..7a9d29b02 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -55,7 +55,7 @@ def remove(self, port, protocol):
 
     @property
     def addresses(self):
-        data = self.scheduler().gateways.get(self.app.id, self.name, ignore_exception=True)
+        data = self.scheduler.gateways.get(self.app.id, self.name, ignore_exception=True)
         if data.status_code != 200:
             return []
         addresses = data.json()["status"].get("addresses", [])
@@ -98,17 +98,17 @@ def refresh_to_k8s(self):
             kwargs["annotations"] = {"cert-manager.io/issuer": self.app.id}
         try:
             try:
-                data = self.scheduler().gateways.get(self.app.id, self.name).json()
+                data = self.scheduler.gateways.get(self.app.id, self.name).json()
                 if len(kwargs["listeners"]) > 0:
                     kwargs["version"] = data["metadata"]["resourceVersion"]
-                    self.scheduler().gateways.patch(self.app.id, self.name, **kwargs)
+                    self.scheduler.gateways.patch(self.app.id, self.name, **kwargs)
                 else:
                     logger.debug("delete k8s resource when listeners are empty")
-                    self.scheduler().gateways.delete(
+                    self.scheduler.gateways.delete(
                         self.app.id, self.name, ignore_exception=True)
             except KubeException:
                 if len(kwargs["listeners"]) > 0:
-                    self.scheduler().gateways.create(self.app.id, self.name, **kwargs)
+                    self.scheduler.gateways.create(self.app.id, self.name, **kwargs)
                 else:
                     logger.debug("skip creating k8s resource when listeners are empty")
         except KubeException as e:
@@ -131,7 +131,7 @@ def save(self, *args, **kwargs):
 
     def delete(self, *args, **kwargs):
         try:
-            self.scheduler().gateways.delete(self.app.id, self.name, ignore_exception=False)
+            self.scheduler.gateways.delete(self.app.id, self.name, ignore_exception=False)
         except KubeException:
             self.log(
                 'Kubernetes gateway cannot be deleted: {}'.format(self.name),
@@ -265,13 +265,13 @@ def refresh_to_k8s(self):
                 self._https_enforced_to_k8s(http_parent_refs)
             elif self.kind == "HTTPRoute":
                 parent_refs.extend(http_parent_refs)
-                self.scheduler().httproute.delete(self.app.id, self._https_redirect_name)
+                self.scheduler.httproute.delete(self.app.id, self._https_redirect_name)
             else:
                 parent_refs.extend(http_parent_refs)
             self._refresh_to_k8s(self.rules, parent_refs)
         else:
-            self.scheduler().httproute.delete(self.app.id, self.name)
-            self.scheduler().httproute.delete(self.app.id, self._https_redirect_name)
+            self.scheduler.httproute.delete(self.app.id, self.name)
+            self.scheduler.httproute.delete(self.app.id, self._https_redirect_name)
 
     def change_default_tls(self):
         if self.app.id != self.name:
@@ -316,7 +316,7 @@ def save(self, *args, **kwargs):
 
     def delete(self, *args, **kwargs):
         try:
-            k8s_route = getattr(self.scheduler(), self.kind.lower())
+            k8s_route = getattr(self.scheduler, self.kind.lower())
             k8s_route.delete(self.app.id, self.name, ignore_exception=False)
         except KubeException:
             self.log(
@@ -357,7 +357,7 @@ def _refresh_to_k8s(self, rules, parent_refs):
             "parent_refs": parent_refs,
         }
         try:
-            k8s_route = getattr(self.scheduler(), self.kind.lower())
+            k8s_route = getattr(self.scheduler, self.kind.lower())
             try:
                 data = k8s_route.get(self.app.id, self.name).json()
                 manifest.update({"version": data["metadata"]["resourceVersion"]})
@@ -383,13 +383,13 @@ def _https_enforced_to_k8s(self, parent_refs):
         }
         try:
             try:
-                data = self.scheduler().httproute.get(
+                data = self.scheduler.httproute.get(
                     self.app.id, self._https_redirect_name).json()
                 manifest.update({"version": data["metadata"]["resourceVersion"]})
-                self.scheduler().httproute.patch(
+                self.scheduler.httproute.patch(
                     self.app.id, self._https_redirect_name, **manifest)
             except KubeException:
-                self.scheduler().httproute.create(
+                self.scheduler.httproute.create(
                     self.app.id, self._https_redirect_name, **manifest)
         except KubeException as e:
             raise ServiceUnavailable(
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index fb07a810e..68e244437 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -288,7 +288,7 @@ def clean(self, ptypes=None):
             labels['type__in'] = ptypes
         # Cleanup controllers
         replica_sets_removal = []
-        replica_sets = self.scheduler().rs.get(self.app.id, labels=labels).json()['items']
+        replica_sets = self.scheduler.rs.get(self.app.id, labels=labels).json()['items']
         if not replica_sets:
             replica_sets = []
         for replica_set in replica_sets:
@@ -320,11 +320,11 @@ def _cleanup_stray_pods(self, namespace, ptypes, latest_version):
         labels = {'heritage': 'drycc'}
         if ptypes is not None:
             labels['type__in'] = ptypes
-        pods = self.scheduler().pod.get(namespace, labels=labels).json()['items']
+        pods = self.scheduler.pod.get(namespace, labels=labels).json()['items']
         if not pods:
             pods = []
         for pod in pods:
-            if self.scheduler().pod.deleted(pod):
+            if self.scheduler.pod.deleted(pod):
                 continue
 
             current_version = int(pod['metadata']['labels']['version'][1:])
@@ -333,7 +333,7 @@ def _cleanup_stray_pods(self, namespace, ptypes, latest_version):
                 continue
 
             try:
-                self.scheduler().pod.delete(namespace, pod['metadata']['name'])
+                self.scheduler.pod.delete(namespace, pod['metadata']['name'])
             except KubeHTTPException as e:
                 # Sometimes k8s will manage to remove the pod from under us
                 if e.response.status_code == 404:
@@ -352,7 +352,7 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, ptypes=None):
         # Find all ReplicaSets
         version_names = [self.version_name, ]
         labels = {'heritage': 'drycc', 'app': namespace}
-        replicasets = self.scheduler().rs.get(namespace, labels=labels).json()['items']
+        replicasets = self.scheduler.rs.get(namespace, labels=labels).json()['items']
         if not replicasets:
             replicasets = []
         for replicaset in replicasets:
@@ -378,11 +378,11 @@ def _cleanup_deployment_secrets_and_configs(self, namespace, ptypes=None):
         if ptypes is not None:
             labels['type__in'] = ptypes
         self.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
-        secrets = self.scheduler().secret.get(namespace, labels=labels).json()['items']
+        secrets = self.scheduler.secret.get(namespace, labels=labels).json()['items']
         if not secrets:
             secrets = []
         for secret in secrets:
-            self.scheduler().secret.delete(namespace, secret['metadata']['name'])
+            self.scheduler.secret.delete(namespace, secret['metadata']['name'])
 
     def _delete_release_in_scheduler(self, namespace, ptypes, version_name):
         """
@@ -398,7 +398,7 @@ def _delete_release_in_scheduler(self, namespace, ptypes, version_name):
         }
         if ptypes is not None:
             labels['type__in'] = ptypes
-        replica_sets = self.scheduler().rs.get(namespace, labels=labels).json()['items']
+        replica_sets = self.scheduler.rs.get(namespace, labels=labels).json()['items']
         if not replica_sets:
             replica_sets = []
         for replica_set in replica_sets:
@@ -409,8 +409,8 @@ def _delete_release_in_scheduler(self, namespace, ptypes, version_name):
             # Deployment takes care of this in the API, RS does not
             # Have the RS scale down pods and delete itself
             try:
-                self.scheduler().rs.scale(namespace, replica_set['metadata']['name'], 0, timeout)
-                self.scheduler().rs.delete(namespace, replica_set['metadata']['name'])
+                self.scheduler.rs.scale(namespace, replica_set['metadata']['name'], 0, timeout)
+                self.scheduler.rs.delete(namespace, replica_set['metadata']['name'])
             except KubeHTTPException as e:
                 if e.response.status_code != 404:
                     raise
diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index 972b2ff56..10128de37 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -3,7 +3,7 @@
 from django.db import models, transaction
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException, AlreadyExists, ServiceUnavailable
-from api.utils import validate_label
+from api.utils import validate_label, get_scheduler
 from scheduler import KubeException
 from .base import UuidAuditedModel
 
@@ -39,8 +39,8 @@ def save(self, *args, **kwargs):
 
     @classmethod
     def services(cls):
-        services = []
-        for serviceclass in cls.scheduler().svcat.get_serviceclasses().json()["items"]:
+        services, scheduler = [], get_scheduler()
+        for serviceclass in scheduler.svcat.get_serviceclasses().json()["items"]:
             services.append({
                 "id": serviceclass["spec"]["externalID"],
                 "name": serviceclass["spec"]["externalName"],
@@ -51,14 +51,14 @@ def services(cls):
 
     @classmethod
     def plans(cls, serviceclass_name):
-        serviceclass_id = None
+        serviceclass_id, scheduler = None, get_scheduler()
         for service in cls.services():
             if service["name"] == serviceclass_name:
                 serviceclass_id = service["id"]
                 break
         plans = []
         if serviceclass_id is not None:
-            for serviceplan in cls.scheduler().svcat.get_serviceplans().json()["items"]:
+            for serviceplan in scheduler.svcat.get_serviceplans().json()["items"]:
                 if serviceplan["spec"]["clusterServiceClassRef"]["name"] == serviceclass_id:
                     plans.append({
                         "id": serviceplan["spec"]["externalID"],
@@ -74,7 +74,7 @@ def plans(cls, serviceclass_name):
 
     def attach(self, *args, **kwargs):
         try:
-            self.scheduler().svcat.get_instance(self.app.id, self.name)
+            self.scheduler.svcat.get_instance(self.app.id, self.name)
             err = "Resource {} already exists in this namespace".format(self.name)  # noqa
             self.log(err, logging.INFO)
             raise AlreadyExists(err)
@@ -87,7 +87,7 @@ def attach(self, *args, **kwargs):
                     "instance_plan": ":".join(instance[1:]),
                     "parameters": self.options,
                 }
-                self.scheduler().svcat.create_instance(
+                self.scheduler.svcat.create_instance(
                     self.app.id, self.name, **kwargs
                 )
             except KubeException as e:
@@ -108,10 +108,10 @@ def delete(self, *args, **kwargs):
 
     def detach(self, *args, **kwargs):
         try:
-            resp = self.scheduler().svcat.get_instance(
+            resp = self.scheduler.svcat.get_instance(
                 self.app.id, self.name, ignore_exception=True)
             if resp.status_code != 404:
-                self.scheduler().svcat.delete_instance(self.app.id, self.name)
+                self.scheduler.svcat.delete_instance(self.app.id, self.name)
         except KubeException as e:
             raise ServiceUnavailable("Could not delete resource {} for application {}".format(self.name, self.app_id)) from e  # noqa
 
@@ -134,14 +134,14 @@ def bind(self, *args, **kwargs):
         self.binding = "Binding"
         self.save()
         try:
-            self.scheduler().svcat.get_binding(self.app.id, self.name)
+            self.scheduler.svcat.get_binding(self.app.id, self.name)
             err = "Resource {} is binding".format(self.name)
             self.log(err, logging.INFO)
             raise AlreadyExists(err)
         except KubeException as e:
             logger.info(e)
             try:
-                self.scheduler().svcat.create_binding(
+                self.scheduler.svcat.create_binding(
                     self.app.id, self.name, **kwargs)
             except KubeException as e:
                 msg = 'There was a problem binding the resource ' \
@@ -153,8 +153,8 @@ def unbind(self, *args, **kwargs):
             raise DryccException("the resource instance is not binding")
         try:
             # We raise an exception when a resource doesn't exist
-            self.scheduler().svcat.get_binding(self.app.id, self.name)
-            self.scheduler().svcat.delete_binding(self.app.id, self.name)
+            self.scheduler.svcat.get_binding(self.app.id, self.name)
+            self.scheduler.svcat.delete_binding(self.app.id, self.name)
             self.binding = None
             self.data = {}
             self.save()
@@ -163,7 +163,7 @@ def unbind(self, *args, **kwargs):
 
     def attach_update(self, *args, **kwargs):
         try:
-            data = self.scheduler().svcat.get_instance(
+            data = self.scheduler.svcat.get_instance(
                 self.app.id, self.name).json()
         except KubeException as e:
             logger.debug(e)
@@ -177,7 +177,7 @@ def attach_update(self, *args, **kwargs):
                 "parameters": self.options,
                 "external_id": data["spec"]["externalID"]
             }
-            self.scheduler().svcat.patch_instance(
+            self.scheduler.svcat.patch_instance(
                 self.app.id, self.name, version, **kwargs
             )
         except KubeException as e:
@@ -188,7 +188,7 @@ def attach_update(self, *args, **kwargs):
     @property
     def message(self):
         try:
-            resp = self.scheduler().svcat.get_instance(
+            resp = self.scheduler.svcat.get_instance(
                 self.app.id, self.name)
             if resp.status_code != 200:
                 message = ""
@@ -222,7 +222,7 @@ def to_measurements(self, timestamp: float):
     def _retrieve_status(self):
         changed = False
         try:
-            response = self.scheduler().svcat.get_instance(
+            response = self.scheduler.svcat.get_instance(
                 self.app.id, self.name).json()
             status = response.get('status', {}).get('lastConditionState')
             options = response.get('spec', {}).get('parameters', {})
@@ -240,14 +240,14 @@ def _retrieve_binding(self):
         changed = False
         try:
             # We raise an exception when a resource doesn't exist
-            response = self.scheduler().svcat.get_binding(self.app.id, self.name).json()
+            response = self.scheduler.svcat.get_binding(self.app.id, self.name).json()
             binding = response.get('status', {}).get('lastConditionState')
             secret_name = response.get('spec', {}).get('secretName')
             if self.binding != binding:
                 self.binding = binding
                 changed = True
             if secret_name:
-                response = self.scheduler().secret.get(self.app.id, secret_name).json()
+                response = self.scheduler.secret.get(self.app.id, secret_name).json()
                 data = response.get('data', {})
                 if self.data != data:
                     self.data = data
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index d2958e8df..47a46dc17 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -140,14 +140,14 @@ def refresh_k8s_svc(self):
         self.log('creating service: {}'.format(self.name), level=logging.DEBUG)
         try:
             try:
-                data = self.scheduler().svc.get(self.namespace, self.name).json()
-                self.scheduler().svc.patch(self.namespace, self.name, **{
+                data = self.scheduler.svc.get(self.namespace, self.name).json()
+                self.scheduler.svc.patch(self.namespace, self.name, **{
                     "ports": self.ports,
                     "version": data["metadata"]["resourceVersion"],
                     "ptype": self.ptype,
                 })
             except KubeException:
-                self.scheduler().svc.create(self.namespace, self.name, **{
+                self.scheduler.svc.create(self.namespace, self.name, **{
                     "ports": self.ports,
                     "ptype": self.ptype,
                 })
@@ -156,4 +156,4 @@ def refresh_k8s_svc(self):
 
     def _delete_k8s_svc(self, svc_name):
         self.log('deleting Service: {}'.format(svc_name), level=logging.DEBUG)
-        self.scheduler().svc.delete(self.namespace, svc_name, ignore_exception=True)
+        self.scheduler.svc.delete(self.namespace, svc_name, ignore_exception=True)
diff --git a/rootfs/api/models/tls.py b/rootfs/api/models/tls.py
index 828269f69..8eb98b1c2 100644
--- a/rootfs/api/models/tls.py
+++ b/rootfs/api/models/tls.py
@@ -54,16 +54,16 @@ def to_result(name, kind, condition):
 
         results = []
         name = namespace = self.app.id
-        response = self.scheduler().issuer.get(namespace, name, ignore_exception=True)
+        response = self.scheduler.issuer.get(namespace, name, ignore_exception=True)
         if response.status_code == 200:
             for condition in response.json()["status"]["conditions"]:
                 results.append(to_result(name, "Issuer", condition))
         name = f"{self.app.id}-auto-tls"
-        response = self.scheduler().certificate.get(namespace, name, ignore_exception=True)
+        response = self.scheduler.certificate.get(namespace, name, ignore_exception=True)
         if response.status_code == 200:
             for condition in response.json()["status"]["conditions"]:
                 results.append(to_result(name, "Certificate", condition))
-        response = self.scheduler().certificaterequest.get(namespace, ignore_exception=True)
+        response = self.scheduler.certificaterequest.get(namespace, ignore_exception=True)
         if response.status_code == 200:
             for item in response.json()["items"]:
                 for condition in item["status"]["conditions"]:
@@ -86,12 +86,12 @@ def refresh_issuer_to_k8s(self):
                 for gateway in self.app.gateway_set.all()
             ]
             try:
-                version = self.scheduler().issuer.get(
+                version = self.scheduler.issuer.get(
                     namespace, name, ignore_exception=False).json()["metadata"]["resourceVersion"]
                 data.update({"version": version})
-                self.scheduler().issuer.put(namespace, name, **data)
+                self.scheduler.issuer.put(namespace, name, **data)
             except KubeException:
-                self.scheduler().issuer.create(namespace, name, **data)
+                self.scheduler.issuer.create(namespace, name, **data)
         except KubeException as e:
             raise ServiceUnavailable('Kubernetes issuer could not be created') from e
 
@@ -100,19 +100,19 @@ def refresh_certificate_to_k8s(self):
         if self.certs_auto_enabled:
             hosts = [domain.domain for domain in self.app.domain_set.all()]
             if len(hosts) > 0:
-                response = self.scheduler().certificate.get(namespace, name)
+                response = self.scheduler.certificate.get(namespace, name)
                 if response.status_code == 200:
                     data = response.json()
                     version = data["metadata"]["resourceVersion"]
-                    self.scheduler().certificate.put(namespace, name, hosts, version)
+                    self.scheduler.certificate.put(namespace, name, hosts, version)
                 else:
                     self.log(
                         "certificate {} does not exist".format(namespace), level=logging.INFO)
-                    self.scheduler().certificate.create(namespace, name, hosts)
+                    self.scheduler.certificate.create(namespace, name, hosts)
             else:
                 self.log("skip creating certificate, no domain name set", logging.WARNING)
         else:
-            self.scheduler().certificate.delete(namespace, name, ignore_exception=True)
+            self.scheduler.certificate.delete(namespace, name, ignore_exception=True)
 
     @transaction.atomic
     def save(self, *args, **kwargs):
@@ -147,13 +147,13 @@ def _refresh_secret_to_k8s(self):
         secret_name = f"{self.app.id}-acme-external-account-binding-secret"
         try:
             try:
-                data = self.scheduler().secret.get(self.app.id, secret_name).json()
-                self.scheduler().secret.patch(self.app.id, secret_name, {
+                data = self.scheduler.secret.get(self.app.id, secret_name).json()
+                self.scheduler.secret.patch(self.app.id, secret_name, {
                     "secret": self.issuer["key_secret"],
                     "version": data["metadata"]["resourceVersion"],
                 })
             except KubeException:
-                self.scheduler().secret.create(self.app.id, secret_name, {
+                self.scheduler.secret.create(self.app.id, secret_name, {
                     "secret": self.issuer["key_secret"],
                 })
         except KubeException as e:
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 6821fb037..58f1ae7fe 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -144,16 +144,16 @@ def _save_pv(self):
         t = string.Template(json.dumps(settings.DRYCC_VOLUME_TEMPLATE.get(self.type)))
         kwargs = json.loads(t.safe_substitute(**kwds))
         try:
-            self.scheduler().pv.get(self.pv_name)
+            self.scheduler.pv.get(self.pv_name)
             msg = "Volume {} already exists".format(self.pv_name)
             self.log(msg, logging.INFO)
             if "csi" in kwargs["spec"]:
                 del kwargs["spec"]["csi"]  # Fix: 422 unprocessable Entity
-            self.scheduler().pv.patch(self.pv_name, **kwargs)
+            self.scheduler.pv.patch(self.pv_name, **kwargs)
         except KubeException as e:
             logger.info(e)
             try:
-                self.scheduler().pv.create(self.pv_name, **kwargs)
+                self.scheduler.pv.create(self.pv_name, **kwargs)
             except KubeException as e:
                 msg = 'There was a problem creating the volume ' \
                       '{} for {}'.format(self.pv_name, self.app_id)
@@ -162,8 +162,8 @@ def _save_pv(self):
     def _delete_pv(self):
         try:
             # We raise an exception when a volume doesn't exist
-            self.scheduler().pv.get(self.pv_name)
-            self.scheduler().pv.delete(self.pv_name)
+            self.scheduler.pv.get(self.pv_name)
+            self.scheduler.pv.delete(self.pv_name)
         except KubeException as e:
             logger.exception(e)
 
@@ -177,14 +177,14 @@ def _save_pvc(self):
         t = string.Template(json.dumps(settings.DRYCC_VOLUME_CLAIM_TEMPLATE.get(self.type)))
         kwargs = json.loads(t.safe_substitute(**kwds))
         try:
-            self.scheduler().pvc.get(self.app.id, self.name)
+            self.scheduler.pvc.get(self.app.id, self.name)
             msg = "Volume claim {} already exists in this namespace".format(self.name)
             self.log(msg, logging.INFO)
-            self.scheduler().pvc.patch(self.app.id, self.name, **kwargs)
+            self.scheduler.pvc.patch(self.app.id, self.name, **kwargs)
         except KubeException as e:
             logger.info(e)
             try:
-                self.scheduler().pvc.create(self.app.id, self.name, **kwargs)
+                self.scheduler.pvc.create(self.app.id, self.name, **kwargs)
             except KubeException as e:
                 msg = 'There was a problem creating the volume claim ' \
                       '{} for {}'.format(self.name, self.app_id)
@@ -193,8 +193,8 @@ def _save_pvc(self):
     def _delete_pvc(self):
         try:
             # We raise an exception when a volume doesn't exist
-            self.scheduler().pvc.get(self.app.id, self.name)
-            self.scheduler().pvc.delete(self.app.id, self.name)
+            self.scheduler.pvc.get(self.app.id, self.name)
+            self.scheduler.pvc.delete(self.app.id, self.name)
         except KubeException as e:
             logger.exception(e)
 
@@ -203,14 +203,14 @@ def _save_secret(self):
         t = string.Template(json.dumps(settings.DRYCC_SECRET_TEMPLATE.get(self.type)))
         kwargs = json.loads(t.safe_substitute(**kwds))
         try:
-            self.scheduler().secret.get(self.app.id, self.secret_name)
+            self.scheduler.secret.get(self.app.id, self.secret_name)
             msg = "Secret {} already exists".format(self.secret_name)
             self.log(msg, logging.INFO)
-            self.scheduler().secret.patch(self.app.id, self.secret_name, **kwargs)
+            self.scheduler.secret.patch(self.app.id, self.secret_name, **kwargs)
         except KubeException as e:
             logger.info(e)
             try:
-                self.scheduler().secret.create(self.app.id, self.secret_name, **kwargs)
+                self.scheduler.secret.create(self.app.id, self.secret_name, **kwargs)
             except KubeException as e:
                 msg = 'There was a problem creating the volume ' \
                       '{} for {}'.format(self.secret_name, self.app_id)
@@ -219,8 +219,8 @@ def _save_secret(self):
     def _delete_secret(self):
         try:
             # We raise an exception when a volume doesn't exist
-            self.scheduler().secret.get(self.app.id, self.secret_name)
-            self.scheduler().secret.delete(self.app.id, self.secret_name)
+            self.scheduler.secret.get(self.app.id, self.secret_name)
+            self.scheduler.secret.delete(self.app.id, self.secret_name)
         except KubeException as e:
             logger.exception(e)
 
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 1bd42d31f..39f184997 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -666,14 +666,14 @@ def test_app_settings_change_routable(self):
         )
         self.assertEqual(response.status_code, 201, response.data)
         app = App.objects.get(id=app_id)
-        response = app.scheduler().httproute.get(app_id, route_name, ignore_exception=True)
+        response = app.scheduler.httproute.get(app_id, route_name, ignore_exception=True)
         self.assertEqual(response.status_code, 404)
         # Set routable to false
         response = self.client.post(
             f'/v2/apps/{app_id}/settings',
             {'routable': True}
         )
-        response = app.scheduler().httproute.get(app_id, route_name, ignore_exception=True)
+        response = app.scheduler.httproute.get(app_id, route_name, ignore_exception=True)
         self.assertEqual(response.status_code, 200)
 
     def test_route_delete(self):
@@ -848,11 +848,11 @@ def test_route_parent_refs(self):
         )
         self.assertEqual(response.status_code, 201, response.data)
         route = Route.objects.get(name=route_name)
-        response = route.scheduler().httproutes.get(app_id, route_name)
+        response = route.scheduler.httproutes.get(app_id, route_name)
         self.assertEqual(response.status_code, 200, response.json())
         self.assertEqual(len(response.json()['spec']['parentRefs']), 1)
         # get gateway
-        response = route.scheduler().gateways.get(app_id, app_id)
+        response = route.scheduler.gateways.get(app_id, app_id)
         listeners = [
             listener for listener in response.json()['spec']['listeners']
             if listener['port'] == 443
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index 444fcee8a..a8ea4deef 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -15,6 +15,7 @@
 import pkgutil
 import inspect
 import requests
+import importlib
 import jsonschema
 from copy import deepcopy
 from django.db import models
@@ -47,6 +48,13 @@ def get_session():
     return session
 
 
+def get_scheduler(metadata=None):
+    mod = importlib.import_module(settings.SCHEDULER_MODULE)
+    metadata = metadata if metadata is not None else {}
+    metadata = dict_merge(metadata, {"annotations": {"app.kubernetes.io/managed-by": "drycc"}})
+    return mod.SchedulerClient(settings.SCHEDULER_URL, settings.K8S_API_VERIFY_TLS, metadata)
+
+
 def import_all_models():
     for _, modname, ispkg in pkgutil.iter_modules([
         os.path.join(os.path.dirname(__file__), "models")
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index f72f3c755..36d64b29f 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -957,7 +957,7 @@ class AppFilerClientViewSet(AppResourceViewSet):
     def get_client(self):
         volume = get_object_or_404(
             models.volume.Volume, app=self.get_app(), name=self.kwargs['name'])
-        return filer.FilerClient(volume.app.id, volume, volume.app.scheduler())
+        return filer.FilerClient(volume.app.id, volume, volume.app.scheduler)
 
     def list(self, request, **kwargs):
         path = request.query_params.get('path', '')
diff --git a/rootfs/scheduler/__init__.py b/rootfs/scheduler/__init__.py
index 82aa8dadf..94c51c6f7 100644
--- a/rootfs/scheduler/__init__.py
+++ b/rootfs/scheduler/__init__.py
@@ -8,7 +8,7 @@
 import re
 from urllib.parse import urljoin
 
-from api import __version__ as drycc_version
+from api import utils, __version__ as drycc_version
 from scheduler.exceptions import KubeException, KubeHTTPException
 
 
@@ -41,10 +41,11 @@ class KubeHTTPClient(object):
     DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
     resource_mapping = OrderedDict()
 
-    def __init__(self, url, k8s_api_verify_tls=True):
+    def __init__(self, url, k8s_api_verify_tls=True, metadata=None):
         self.url = url
         self.k8s_api_verify_tls = k8s_api_verify_tls
         self.session = get_k8s_session(self.k8s_api_verify_tls)
+        self.metadata = metadata
 
         # map the various k8s Resources to an internal property
         from scheduler.resources import Resource  # lazy load
@@ -190,6 +191,8 @@ def http_post(self, path, data=None, json=None, **kwargs):
         """
         try:
             url = urljoin(self.url, path)
+            if self.metadata is not None and "metadata" in data:
+                data["metadata"] = utils.dict_merge(data["metadata"], self.metadata)
             response = self.session.post(url, data=data, json=json, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # reraise as KubeException, but log stacktrace.
@@ -207,6 +210,8 @@ def http_put(self, path, data=None, **kwargs):
         """
         try:
             url = urljoin(self.url, path)
+            if self.metadata is not None and "metadata" in data:
+                data["metadata"] = utils.dict_merge(data["metadata"], self.metadata)
             response = self.session.put(url, data=data, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # reraise as KubeException, but log stacktrace.
@@ -229,6 +234,8 @@ def http_patch(self, path, data=None, **kwargs):
             # application/merge-patch+json,
             # application/apply-patch+yaml
             # self.session.headers["Content-Type"] = "application/json-patch+json"
+            if self.metadata is not None and "metadata" in data:
+                data["metadata"] = utils.dict_merge(data["metadata"], self.metadata)
             response = self.session.patch(url, data=data, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # reraise as KubeException, but log stacktrace.
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index d19b70854..6d1ac5e62 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -1071,8 +1071,8 @@ def session():
 
 
 class MockSchedulerClient(KubeHTTPClient):
-    def __init__(self, url, k8s_api_verify_tls=True):
-        super().__init__(url)
+    def __init__(self, url, k8s_api_verify_tls=True, metadata=None):
+        super().__init__(url, k8s_api_verify_tls, metadata)
 
         # set version data
         cache.set('version', {'major': '1', 'minor': '3'}, None)

From 65631a0936e96ebc8589358e10c86a1c08b26394 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 29 May 2025 16:32:00 +0800
Subject: [PATCH 136/200] chore(metrics): add metrics proxy

---
 rootfs/api/settings/production.py |  5 +-
 rootfs/api/urls.py                |  4 ++
 rootfs/api/views.py               | 76 ++++++++++++++++++++++++++++++-
 rootfs/requirements.txt           |  5 +-
 4 files changed, 82 insertions(+), 8 deletions(-)

diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 8d6021c97..ce6fbd0ad 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -468,11 +468,8 @@
 # Cache Valkey Configuration
 CACHES = {
     "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
         "LOCATION": os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379'),
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.DefaultClient",
-        }
     }
 }
 
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 698722b9d..4a12806b0 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -243,6 +243,10 @@
     re_path(
         r"^apps/(?P<id>{})/perms/(?P<username>[\w.@+-]+)/?$".format(settings.APP_URL_REGEX),
         views.AppPermViewSet.as_view({'put': 'update', 'delete': 'destroy'})),
+    # nodes
+    re_path(
+        r'^nodes/(?P<node>[a-zA-Z0-9-]+)/proxy/metrics(?:/(?P<metrics>[^/]+))?/?$',
+        views.ProxyMetricsView.as_view()),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
     re_path(r"^tokens/(?P<pk>[-_\w]+)/?$", views.TokenViewSet.as_view({'delete': 'destroy'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 36d64b29f..097bc01be 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -5,7 +5,12 @@
 import uuid
 import logging
 import json
+import ssl
+import time
+import random
+import aiohttp
 import requests
+
 from asgiref.sync import async_to_sync
 from django.db.models import Q
 from django.core.cache import cache
@@ -25,7 +30,7 @@
 from rest_framework.viewsets import GenericViewSet
 from rest_framework.exceptions import PermissionDenied
 
-from api import monitor, models, permissions, serializers, viewsets, authentication
+from api import monitor, models, permissions, serializers, viewsets, authentication, __version__
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
     delete_pod
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
@@ -1242,3 +1247,72 @@ def metric(self, request, **kwargs):
         return StreamingHttpResponse(
             streaming_content=monitor.last_metrics(app_id)
         )
+
+
+class ProxyMetricsView(View):
+    cache = {}
+    match_meta = staticmethod(
+        re.compile(r'^(?:# (?:HELP|TYPE) )([a-zA-Z_][a-zA-Z0-9_:.-]*)').match)
+    match_data = staticmethod(
+        re.compile(r'^([a-zA-Z_][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)').match)
+    default_cache_value = (-1, -1)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if settings.K8S_API_VERIFY_TLS:
+            ssl_context = ssl.create_default_context(
+                cafile='/var/run/secrets/kubernetes.io/serviceaccount/ca.crt')
+        else:
+            ssl_context = ssl.create_default_context()
+        self.connector = aiohttp.TCPConnector(ssl_context=ssl_context)
+
+    async def sample(self, name, labels_str, value):
+        if not labels_str or name not in settings.DRYCC_METRICS_CONFIG:
+            return None
+        fields = set(settings.DRYCC_METRICS_CONFIG[name])
+        labels = {}
+        for pair in labels_str.strip(" {}").split(','):
+            k, v = pair.split('=', 1)
+            if k in fields:
+                labels[k] = v.strip(' "')
+        app_id = labels.get("namespace", None)
+        if not app_id:
+            return None
+        owner_id, timeout = self.cache.get(app_id, self.default_cache_value)
+        if (owner_id < 0 and timeout < 0) or time.time() > timeout:
+            if app := await models.app.App.objects.filter(id=app_id).afirst():
+                owner_id = app.owner_id
+            else:
+                owner_id = -1
+            self.cache[app_id] = (owner_id, time.time() + random.randint(600, 1200))
+        if owner_id < 0:
+            return None
+        labels.update({'vm_project_id': app_id, 'vm_account_id': owner_id})
+        return "%s{%s} %s\n" % (name, ",".join([f'{k}="{v}"' for k, v in labels.items()]), value)
+
+    async def get(self, request, node, metrics=None):
+        if not metrics:
+            url = f"{settings.SCHEDULER_URL}/api/v1/nodes/{node}/proxy/metrics"
+        else:
+            url = f"{settings.SCHEDULER_URL}/api/v1/nodes/{node}/proxy/metrics/{metrics}"
+        headers = {"Authorization": request.META.get("HTTP_AUTHORIZATION", "")}
+
+        async def stream_response():
+            async with aiohttp.ClientSession(connector=self.connector) as session:
+                async with session.get(url, headers=headers) as resp:
+                    async for line_bytes in resp.content:
+                        line = line_bytes.decode('utf-8', errors='ignore').strip(' \n')
+                        if line.startswith('#') and (match := self.match_meta(line)):
+                            if match.group(1) in settings.DRYCC_METRICS_CONFIG:
+                                yield f"{line}\n"
+                            continue
+                        match = self.match_data(line)
+                        if not match:
+                            continue
+                        name, labels_str, value = match.groups()
+                        sample = await self.sample(name, labels_str, value)
+                        if not sample:
+                            continue
+                        yield sample
+        content_type = f"text/plain; version={__version__}"
+        return StreamingHttpResponse(stream_response(), content_type=content_type)
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 2d2bca86c..711c190ba 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -22,9 +22,8 @@ ndg-httpsclient==0.5.1
 pytz==2025.2
 requests==2.32.3
 requests-toolbelt==1.0.0
-celery==5.5.0
-hiredis==3.1.0
-django_redis==5.4.0
+celery[redis]==5.5.0
+hiredis==3.2.1
 dj-database-url==2.3.0
 social-auth-app-django==5.4.3
 kubernetes==32.0.1

From a39347927b81f3bc350ce0bafb12fa4420e0ad84 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 9 Jun 2025 17:58:25 +0800
Subject: [PATCH 137/200] feat(controller): add a multi-tenant proxy for
 prometheus api

---
 charts/controller/templates/_helpers.tpl      | 12 +++----
 .../templates/controller-secret-creds.yaml    |  4 +--
 charts/controller/values.yaml                 |  6 ++--
 rootfs/api/authentication.py                  | 20 +++++++----
 .../commands/measure_loadbalancers.py         |  2 +-
 rootfs/api/monitor.py                         | 16 ++++-----
 rootfs/api/settings/production.py             |  4 +--
 rootfs/api/urls.py                            | 15 +++++---
 rootfs/api/views.py                           | 35 ++++++++++++++++++-
 rootfs/requirements.txt                       |  2 +-
 10 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index d4d87618c..7ffaf9d27 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -112,15 +112,15 @@ env:
   valueFrom:
     fieldRef:
       fieldPath: metadata.namespace
-{{- if (.Values.prometheusUrl) }}
-- name: "DRYCC_PROMETHEUS_URL"
+{{- if (.Values.victoriametricsUrl) }}
+- name: "DRYCC_VICTORIAMETRICS_URL"
   valueFrom:
     secretKeyRef:
       name: controller-creds
-      key: prometheus-url
-{{- else if .Values.prometheus.enabled }}
-- name: "DRYCC_PROMETHEUS_URL"
-  value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481/select/0/prometheus"
+      key: victoriametrics-url
+{{- else if .Values.victoriametrics.enabled }}
+- name: "DRYCC_VICTORIAMETRICS_URL"
+  value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index 3b98a9f2f..7afbc8192 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -14,8 +14,8 @@ data:
   {{- if (.Values.databaseReplicaUrl) }}
   database-replica-url: {{ .Values.databaseReplicaUrl | b64enc }}
   {{- end }}
-  {{- if (.Values.prometheusUrl) }}
-  prometheus-url: {{ .Values.prometheusUrl | b64enc }}
+  {{- if (.Values.victoriametricsUrl) }}
+  victoriametrics-url: {{ .Values.victoriametricsUrl | b64enc }}
   {{- end }}
   {{- if (.Values.passportUrl) }}
   passport-url: {{ .Values.passportUrl | b64enc }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 7756dc449..179f08eb0 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -63,8 +63,8 @@ databaseReplicaUrl: ""
 passportUrl: ""
 passportKey: ""
 passportSecret: ""
-# prometheusUrl is will no longer use the built-in prometheus component
-prometheusUrl: ""
+# victoriametricsUrl is will no longer use the built-in victoriametrics component
+victoriametricsUrl: ""
 # Workflow-manager Configuration Options
 workflowManagerUrl: ""
 workflowManagerAccessKey: ""
@@ -169,7 +169,7 @@ registry:
 passport:
   enabled: true
 
-prometheus:
+victoriametrics:
   enabled: true
 
 global:
diff --git a/rootfs/api/authentication.py b/rootfs/api/authentication.py
index 61a9c1f7a..5649fb213 100644
--- a/rootfs/api/authentication.py
+++ b/rootfs/api/authentication.py
@@ -45,8 +45,11 @@ def authenticate(self, request):
         if token_type is None or token is None:
             return None
         if token_type == 'bearer':  # drycc oauth access token
-            from api.backend import OauthCacheManager
-            return OauthCacheManager().get_user(token), token
+            try:
+                from api.backend import OauthCacheManager
+                return OauthCacheManager().get_user(token), token
+            except exceptions.AuthenticationFailed:
+                return None, None
         # drycc token
         user = cache.get(token, None)
         if not user:
@@ -62,11 +65,14 @@ def authenticate_credentials(self, key):
         if not token.owner.is_active:
             raise exceptions.AuthenticationFailed(gettext_lazy('User inactive or deleted.'))
         if token.expires():
-            from api.backend import OauthCacheManager
-            token.refresh_token()
-            user = OauthCacheManager().get_user(token.oauth['access_token'])
-            cache.set(key, user, timeout=token.oauth['expires_in'])
-            return user, token.key
+            try:
+                from api.backend import OauthCacheManager
+                user = OauthCacheManager().get_user(token.oauth['access_token'])
+                cache.set(key, user, timeout=token.oauth['expires_in'])
+                token.refresh_token()
+                return user, token.key
+            except exceptions.AuthenticationFailed:
+                return None, None
         return (token.owner, token.key)
 
     def authenticate_header(self, request):
diff --git a/rootfs/api/management/commands/measure_loadbalancers.py b/rootfs/api/management/commands/measure_loadbalancers.py
index 277bdc6a8..9997ca3ba 100644
--- a/rootfs/api/management/commands/measure_loadbalancers.py
+++ b/rootfs/api/management/commands/measure_loadbalancers.py
@@ -48,7 +48,7 @@ def _measure_loadbalancers(self, app_map, timestamp):
         send_measurements.delay(loadbalancers)
 
     def handle(self, *args, **options):
-        if settings.WORKFLOW_MANAGER_URL and settings.DRYCC_PROMETHEUS_URL:
+        if settings.WORKFLOW_MANAGER_URL and settings.DRYCC_VICTORIAMETRICS_URL:
             timestamp = int(time.time())
             task_id = uuid.uuid4().hex
             logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 2221d9e61..71d498eb7 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -58,7 +58,7 @@ async def query_prom(url, params) -> list[tuple[dict[str, str], int]]:
 async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
     if not settings.DRYCC_METRICS_CONFIG:
         return
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
     promql = query_last_metrics_promql_tpl % (
       '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
       namespace,
@@ -76,21 +76,21 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
 
 async def query_loadbalancer(namespaces: Iterator[str], start: int, stop: int
                              ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
     promql = query_loadbalancer_promql_tpl % "|".join(namespaces)
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
 
 async def query_network_receive_flow(namespaces: Iterator[str], start: int, stop: int
                                      ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
     promql = query_network_receive_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
 
 async def query_network_transmit_flow(namespaces: Iterator[str], start: int, stop: int
                                       ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
     promql = query_network_transmit_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
@@ -98,7 +98,7 @@ async def query_network_transmit_flow(namespaces: Iterator[str], start: int, sto
 async def query_cpu_usage(namespace: str, ptype: str, every: str,
                           start: int, stop: int, step: int,
                           ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_cpu_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
@@ -107,7 +107,7 @@ async def query_cpu_usage(namespace: str, ptype: str, every: str,
 async def query_memory_usage(namespace: str, ptype: str, every: str,
                              start: int, stop: int, step: int,
                              ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_memory_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
@@ -116,7 +116,7 @@ async def query_memory_usage(namespace: str, ptype: str, every: str,
 async def query_network_receive_usage(namespace: str, ptype: str, every: str,
                                       start: int, stop: int, step: int,
                                       ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_network_receive_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
@@ -125,7 +125,7 @@ async def query_network_receive_usage(namespace: str, ptype: str, every: str,
 async def query_network_transmit_usage(namespace: str, ptype: str, every: str,
                                        start: int, stop: int, step: int,
                                        ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query_range")
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_network_transmit_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index ce6fbd0ad..5954791a9 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -269,8 +269,8 @@
 
 K8S_API_VERIFY_TLS = os.environ.get('K8S_API_VERIFY_TLS', 'true').lower() == "true"
 
-# drycc prometheus url
-DRYCC_PROMETHEUS_URL = os.environ.get('DRYCC_PROMETHEUS_URL', '')
+# drycc victoriametrics url
+DRYCC_VICTORIAMETRICS_URL = os.environ.get('DRYCC_VICTORIAMETRICS_URL', '')
 
 # drycc metrics config file
 DRYCC_METRICS_CONFIG = {}
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 4a12806b0..910d8c467 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -247,13 +247,11 @@
     re_path(
         r'^nodes/(?P<node>[a-zA-Z0-9-]+)/proxy/metrics(?:/(?P<metrics>[^/]+))?/?$',
         views.ProxyMetricsView.as_view()),
+    # prometheus
+    re_path(r'^prometheus/(?P<path>.+)/?$', views.PrometheusProxy.as_view()),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
     re_path(r"^tokens/(?P<pk>[-_\w]+)/?$", views.TokenViewSet.as_view({'delete': 'destroy'})),
-    # social login is placed at the end of the URL match
-    re_path(r'^login/(?P<backend>[^/]+){0}$'.format(extra), views.auth, name='begin'),
-    re_path(r'^complete/(?P<backend>[^/]+){0}$'.format(extra), views.complete, name='complete'),
-    re_path('', include('social_django.urls', namespace='social')),
 ]
 
 mutate_urlpatterns = [
@@ -263,8 +261,15 @@
     ),
 ]
 
+# social login is placed at the end of the URL match
+social_urlpatterns = [
+    re_path(r'^login/(?P<backend>[^/]+){0}$'.format(extra), views.auth, name='begin'),
+    re_path(r'^complete/(?P<backend>[^/]+){0}$'.format(extra), views.complete, name='complete'),
+    re_path('', include('social_django.urls', namespace='social')),
+]
+
 # If there is a mutating admission mutate configuration, use mutate url
 if settings.MUTATE_KEY:
     urlpatterns = mutate_urlpatterns
 else:
-    urlpatterns = app_urlpatterns
+    urlpatterns = app_urlpatterns + social_urlpatterns
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 097bc01be..0de94f7e5 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -10,7 +10,9 @@
 import random
 import aiohttp
 import requests
+import warnings
 
+from urllib.parse import urljoin
 from asgiref.sync import async_to_sync
 from django.db.models import Q
 from django.core.cache import cache
@@ -38,7 +40,8 @@
 from django.views.decorators.cache import never_cache
 from django.contrib.auth import REDIRECT_FIELD_NAME
 from django.views.decorators.csrf import csrf_exempt
-from django.http.response import FileResponse, StreamingHttpResponse
+from django.http.response import FileResponse, JsonResponse, StreamingHttpResponse
+from channels.db import database_sync_to_async
 from social_django.utils import psa
 from social_django.views import _do_login
 from social_core.utils import setting_name
@@ -1217,6 +1220,8 @@ def wrap(app_id, ptype, every, start, stop, step):
     @method_decorator(cache_page(settings.DRYCC_METRICS_EXPIRY))
     @method_decorator(vary_on_headers("Authorization"))
     def status(self, request, **kwargs):
+        warnings.warn(
+            'this interface will be removed in the next version.', PendingDeprecationWarning)
         app_id = self._get_app().id
         data = serializers.MetricSerializer(data=self.request.query_params)
         if not data.is_valid():
@@ -1243,6 +1248,8 @@ def status(self, request, **kwargs):
     @method_decorator(cache_page(settings.DRYCC_METRICS_EXPIRY))
     @method_decorator(vary_on_headers("Authorization"))
     def metric(self, request, **kwargs):
+        warnings.warn(
+            'this interface will be removed in the next version.', PendingDeprecationWarning)
         app_id = self._get_app().id
         return StreamingHttpResponse(
             streaming_content=monitor.last_metrics(app_id)
@@ -1316,3 +1323,29 @@ async def stream_response():
                         yield sample
         content_type = f"text/plain; version={__version__}"
         return StreamingHttpResponse(stream_response(), content_type=content_type)
+
+
+@method_decorator(csrf_exempt, name='dispatch')
+class PrometheusProxy(View):
+    timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
+    authentication = authentication.DryccAuthentication()
+
+    async def proxy(self, request, path):
+        auth = await database_sync_to_async(self.authentication.authenticate)(request)
+        if not auth or len(auth) != 2 or not isinstance(auth[0], User):
+            return JsonResponse({'error': 'access denied'}, status=403)
+        if auth[0].is_superuser or auth[0].is_staff:
+            path = f"/select/0/prometheus/{path}"
+        else:
+            path = f"/select/{auth[0].id}/prometheus/{path}"
+        url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, path)
+        params = dict(request.GET) if request.method == "GET" else dict(request.POST)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'victoriametrics connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    get = post = proxy
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 711c190ba..8a4e9085b 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.21
+django==4.2.22
 channels==4.2.2
 aiohttp==3.11.16
 django-cors-headers==4.7.0

From 9837d48c25324011a30e9029bddb0876f1ac5575 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 10 Jun 2025 22:55:41 +0800
Subject: [PATCH 138/200] chore(deps): bump requests from 2.32.3 to 2.32.4 in
 /rootfs (#184)

Bumps [requests](https://github.com/psf/requests) from 2.32.3 to 2.32.4.
- [Release notes](https://github.com/psf/requests/releases)
- [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md)
- [Commits](https://github.com/psf/requests/compare/v2.32.3...v2.32.4)

---
updated-dependencies:
- dependency-name: requests
  dependency-version: 2.32.4
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 8a4e9085b..fc0eaf837 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -20,7 +20,7 @@ psycopg[binary]==3.2.6
 pyOpenSSL==25.0.0
 ndg-httpsclient==0.5.1
 pytz==2025.2
-requests==2.32.3
+requests==2.32.4
 requests-toolbelt==1.0.0
 celery[redis]==5.5.0
 hiredis==3.2.1

From fc26cbd5b83eaf733b5478246485a574c3a6880b Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 12 Jun 2025 11:50:18 +0800
Subject: [PATCH 139/200] fix(authentication): failure exception

---
 rootfs/api/authentication.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/authentication.py b/rootfs/api/authentication.py
index 5649fb213..34d7b0ce7 100644
--- a/rootfs/api/authentication.py
+++ b/rootfs/api/authentication.py
@@ -49,7 +49,7 @@ def authenticate(self, request):
                 from api.backend import OauthCacheManager
                 return OauthCacheManager().get_user(token), token
             except exceptions.AuthenticationFailed:
-                return None, None
+                return None
         # drycc token
         user = cache.get(token, None)
         if not user:

From fd2b96a65b449a42e32762fa4e9c7b67515a06c7 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 12 Jun 2025 13:03:34 +0800
Subject: [PATCH 140/200] feat(token): oauth2 token can be exchanged for a
 controller token

---
 rootfs/api/urls.py  |  1 +
 rootfs/api/views.py | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 910d8c467..01328fff9 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -15,6 +15,7 @@
 app_urlpatterns = [
     re_path(r'^', include(router.urls)),
     re_path(r'auth/login/?$', views.AuthLoginView.as_view({"post": "login"})),
+    re_path(r'auth/token/?$', views.AuthTokenView.as_view({"post": "token"})),
     re_path(r'auth/token/(?P<key>[-_\w]+)/?$', views.AuthTokenView.as_view({"get": "token"})),
     # limits
     re_path(
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 0de94f7e5..f40b4a0b2 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -144,8 +144,14 @@ class AuthTokenView(GenericViewSet):
     permission_classes = (AllowAny, )
 
     def token(self, request, *args, **kwargs):
-        oauth = oauth_cache_manager.get_token(self.kwargs['key'])
-        if oauth:
+        if 'key' in self.kwargs:
+            oauth = oauth_cache_manager.get_token(self.kwargs['key'])
+        else:
+            try:
+                oauth = json.loads(request.body.decode("utf8"))
+            except json.decoder.JSONDecodeError:
+                return HttpResponse(status=400)
+        if oauth and 'access_token' in oauth:
             user = oauth_cache_manager.get_user(oauth['access_token'])
             alias = request.query_params.get('alias', '')
             token = models.base.Token(owner=user, alias=alias, oauth=oauth)

From b956bfdc01ad8acae3ca8dd0b53be76e3b95aab0 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 13 Jun 2025 09:58:28 +0800
Subject: [PATCH 141/200] chore(prometheus): add username to prometheus url

---
 rootfs/api/urls.py  | 3 ++-
 rootfs/api/views.py | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 01328fff9..d6fb21549 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -249,7 +249,8 @@
         r'^nodes/(?P<node>[a-zA-Z0-9-]+)/proxy/metrics(?:/(?P<metrics>[^/]+))?/?$',
         views.ProxyMetricsView.as_view()),
     # prometheus
-    re_path(r'^prometheus/(?P<path>.+)/?$', views.PrometheusProxy.as_view()),
+    re_path(
+        r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$', views.PrometheusProxy.as_view()),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
     re_path(r"^tokens/(?P<pk>[-_\w]+)/?$", views.TokenViewSet.as_view({'delete': 'destroy'})),
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index f40b4a0b2..d589f5a68 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1336,9 +1336,9 @@ class PrometheusProxy(View):
     timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
     authentication = authentication.DryccAuthentication()
 
-    async def proxy(self, request, path):
+    async def proxy(self, request, username, path):
         auth = await database_sync_to_async(self.authentication.authenticate)(request)
-        if not auth or len(auth) != 2 or not isinstance(auth[0], User):
+        if not auth or len(auth) != 2 or not auth[0].username != username:
             return JsonResponse({'error': 'access denied'}, status=403)
         if auth[0].is_superuser or auth[0].is_staff:
             path = f"/select/0/prometheus/{path}"

From d5813900aa9e304b38bc9a4f8b30b6f014e45268 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 13 Jun 2025 10:46:10 +0800
Subject: [PATCH 142/200] feat(token): add get token by uuid api

---
 charts/controller/templates/_helpers.tpl | 12 +++++++++++-
 rootfs/api/urls.py                       |  4 +++-
 rootfs/api/views.py                      | 11 ++++++-----
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 7ffaf9d27..e90c1ed16 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -119,8 +119,18 @@ env:
       name: controller-creds
       key: victoriametrics-url
 {{- else if .Values.victoriametrics.enabled }}
+- name: "DRYCC_VICTORIAMETRICS_USERNAME"
+  valueFrom:
+    secretKeyRef:
+      name: victoriametrics-vmauth-creds
+      key: username
+- name: "DRYCC_VICTORIAMETRICS_PASSWORD"
+  valueFrom:
+    secretKeyRef:
+      name: victoriametrics-vmauth-creds
+      key: password
 - name: "DRYCC_VICTORIAMETRICS_URL"
-  value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481"
+  value: "http://$(DRYCC_VICTORIAMETRICS_USERNAME):$(DRYCC_VICTORIAMETRICS_PASSWORD)@drycc-victoriametrics-vmauth.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index d6fb21549..fe5c662b7 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -253,7 +253,9 @@
         r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$', views.PrometheusProxy.as_view()),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
-    re_path(r"^tokens/(?P<pk>[-_\w]+)/?$", views.TokenViewSet.as_view({'delete': 'destroy'})),
+    re_path(
+        r"^tokens/(?P<pk>[-_\w]+)/?$",
+        views.TokenViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
 ]
 
 mutate_urlpatterns = [
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index d589f5a68..c69a402ab 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -156,7 +156,8 @@ def token(self, request, *args, **kwargs):
             alias = request.query_params.get('alias', '')
             token = models.base.Token(owner=user, alias=alias, oauth=oauth)
             token.save()
-            return HttpResponse(json.dumps({"token": token.key, "username": user.username}))
+            return HttpResponse(json.dumps(
+                {"uuid": str(token.uuid), "token": token.key, "username": user.username}))
         return HttpResponse(status=404)
 
 
@@ -1338,10 +1339,10 @@ class PrometheusProxy(View):
 
     async def proxy(self, request, username, path):
         auth = await database_sync_to_async(self.authentication.authenticate)(request)
-        if not auth or len(auth) != 2 or not auth[0].username != username:
-            return JsonResponse({'error': 'access denied'}, status=403)
-        if auth[0].is_superuser or auth[0].is_staff:
-            path = f"/select/0/prometheus/{path}"
+        if not auth or len(auth) != 2:
+            return JsonResponse({'error': 'Unauthorized'}, status=401)
+        if auth[0].username != username:
+            return JsonResponse({'error': 'Access denied'}, status=403)
         else:
             path = f"/select/{auth[0].id}/prometheus/{path}"
         url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, path)

From 721c06aed3d4feb5d46547ab68155dc3d119b114 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 19 Jun 2025 13:34:50 +0800
Subject: [PATCH 143/200] chore(controller): add reserved names config

---
 charts/controller/templates/_helpers.tpl      | 26 +++++++++++++++++--
 .../templates/controller-configmap.yaml       |  6 +++++
 charts/controller/values.yaml                 |  4 +--
 rootfs/api/models/app.py                      |  4 +--
 rootfs/api/settings/production.py             |  8 +++++-
 rootfs/api/tests/test_app.py                  |  6 ++---
 rootfs/api/tests/test_release.py              |  2 +-
 7 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index e90c1ed16..7cc23cb60 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -130,7 +130,7 @@ env:
       name: victoriametrics-vmauth-creds
       key: password
 - name: "DRYCC_VICTORIAMETRICS_URL"
-  value: "http://$(DRYCC_VICTORIAMETRICS_USERNAME):$(DRYCC_VICTORIAMETRICS_PASSWORD)@drycc-victoriametrics-vmauth.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481"
+  value: "http://$(DRYCC_VICTORIAMETRICS_USERNAME):$(DRYCC_VICTORIAMETRICS_PASSWORD)@drycc-victoriametrics-vmauth.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8427"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
@@ -172,7 +172,6 @@ env:
 {{- end }}
 {{- end }}
 
-
 {{- define "controller-job.envs" }}
 {{- include "controller.envs" . }}
 - name: DRYCC_DATABASE_ROUTERS
@@ -329,6 +328,29 @@ kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim, job]
 {{- end }}
 
+{{/* Generate controller config default reserved names */}}
+{{ define "controller.config.defaultReservedNames" }}
+backup
+catalog
+cert-manager
+default
+drycc
+drycc-manager
+drycc-helmbroker
+drycc-builder
+drycc-grafana
+drycc-passport
+istio-gateway
+istio-system
+kube-node-lease
+kube-public
+kube-system
+longhorn-system
+metallb
+mount-s3
+topolvm
+rook-ceph
+{{- end }}
 
 {{/* Generate controller config default secrets template */}}
 {{ define "controller.config.defaultSecretTemplate" }}
diff --git a/charts/controller/templates/controller-configmap.yaml b/charts/controller/templates/controller-configmap.yaml
index 542cba653..15476dcbe 100644
--- a/charts/controller/templates/controller-configmap.yaml
+++ b/charts/controller/templates/controller-configmap.yaml
@@ -23,6 +23,12 @@ data:
     {{- else}}
     {{- include "controller.config.defaultLimitPlans" . | fromYamlArray | toPrettyJson | nindent 4 }}
     {{- end }}
+  reserved-names.txt: |-
+    {{- if .Values.config.reservedNames }}
+    {{- (tpl .Values.config.reservedNames $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultReservedNames" . | nindent 4 }}
+    {{- end }}
   secret-template.json: |
     {{- if .Values.config.secretTemplate }}
     {{- (tpl .Values.config.secretTemplate $)  | nindent 4 }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 179f08eb0..7cb1d6132 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -76,6 +76,7 @@ config:
   metrics: ""
   limitSpecs: ""
   limitPlans: ""
+  reservedNames: ""
   secretTemplate: ""
   volumeTemplate: ""
   volumeClaimTemplate: ""
@@ -88,8 +89,7 @@ service:
 # Any custom controller environment variables
 # can be specified as key-value pairs under environment
 # this is usually a non required setting.
-environment:
-  RESERVED_NAMES: "drycc, drycc-builder, drycc-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
+environment: {}
 
 api:
   resources: {}
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 7dd2fa176..57aec8ce6 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -84,7 +84,7 @@ def validate_app_id(value):
     """
     Check that the value follows the kubernetes name constraints
     """
-    match = re.match(r'[a-z]([a-z0-9-]*[a-z0-9])?$', value)
+    match = re.match(r'^[a-z]([a-z0-9-]{3,}[a-z0-9])$', value)
     if not match:
         raise ValidationError("App name must start with an alphabetic character, cannot end with a"
                               + " hyphen and can only contain a-z (lowercase), 0-9 and hyphens.")
@@ -101,7 +101,7 @@ def validate_app_structure(value):
 
 def validate_reserved_names(value):
     """A value cannot use some reserved names."""
-    if value in settings.DRYCC_RESERVED_NAMES:
+    if value in settings.RESERVED_NAMES:
         raise ValidationError('{} is a reserved name.'.format(value))
 
 
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 5954791a9..da82fe81c 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -248,7 +248,13 @@
 TEMPDIR = tempfile.mkdtemp(prefix='drycc')
 
 # names which apps cannot reserve for routing
-DRYCC_RESERVED_NAMES = os.environ.get('RESERVED_NAMES', '').replace(' ', '').split(',')
+RESERVED_NAMES_PATH = os.environ.get(
+    'RESERVED_NAMES_PATH', '/etc/controller/reserved-names.txt')
+if os.path.exists(RESERVED_NAMES_PATH):
+    with open(RESERVED_NAMES_PATH) as f:
+        RESERVED_NAMES = [line.strip() for line in f if line]
+else:
+    RESERVED_NAMES = ["drycc", "drycc-helmbroker", "drycc-manager", "kube-system", "default"]
 
 # the k8s namespace in which the controller and workflow were installed.
 WORKFLOW_NAMESPACE = os.environ.get('WORKFLOW_NAMESPACE', 'drycc')
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 77ffa6aed..e36e1d370 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -159,7 +159,7 @@ def test_app_errors(self, mock_requests):
     def test_app_reserved_names(self, mock_requests):
         """Nobody should be able to create applications with names which are reserved."""
         reserved_names = ['foo', 'bar']
-        with self.settings(DRYCC_RESERVED_NAMES=reserved_names):
+        with self.settings(RESERVED_NAMES=reserved_names):
             for name in reserved_names:
                 response = self.client.post('/v2/apps', {'id': name})
                 self.assertContains(
@@ -531,7 +531,7 @@ def test_list_ordering(self, mock_requests):
         """
         Test that a list of apps is sorted by name
         """
-        for name in ['zulu', 'tango', 'alpha', 'foxtrot']:
+        for name in ['zulua', 'tango', 'alpha', 'foxtrot']:
             response = self.client.post('/v2/apps', {'id': name})
             self.assertEqual(response.status_code, 201, response.data)
 
@@ -540,7 +540,7 @@ def test_list_ordering(self, mock_requests):
         self.assertEqual(apps[0]['id'], 'alpha')
         self.assertEqual(apps[1]['id'], 'foxtrot')
         self.assertEqual(apps[2]['id'], 'tango')
-        self.assertEqual(apps[3]['id'], 'zulu')
+        self.assertEqual(apps[3]['id'], 'zulua')
 
     def test_get_private_registry_config(self, mock_requests):
         registry = {"web": {'username': 'test', 'password': 'test'}}
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 8f405702f..4d6f974fd 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -614,7 +614,7 @@ def test_deploy_hooks_logged(self, mock_requests, mock_logger):
         """
         Verifies that a configured deploy hook is dumped into the logs when a release is created.
         """
-        app_id = 'foo'
+        app_id = 'foooo'
         body = {'sha': '123456', 'image': 'autotest/example', 'stack': 'heroku-18'}
 
         mr_rocks = mock_requests.post(f'http://drycc.rocks?app={app_id}&user={self.user.username}&sha=&release=v1&release_summary={self.user.username}+created+initial+release')  # noqa

From da8f91c789c51f84239345ef020aef619a8dcd0a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 24 Jun 2025 17:58:14 +0800
Subject: [PATCH 144/200] chore(views): standardized naming

---
 rootfs/api/urls.py  | 5 +++--
 rootfs/api/views.py | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index fe5c662b7..898c777e4 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -247,10 +247,11 @@
     # nodes
     re_path(
         r'^nodes/(?P<node>[a-zA-Z0-9-]+)/proxy/metrics(?:/(?P<metrics>[^/]+))?/?$',
-        views.ProxyMetricsView.as_view()),
+        views.MetricsProxyView.as_view()),
     # prometheus
     re_path(
-        r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$', views.PrometheusProxy.as_view()),
+        r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$',
+        views.PrometheusProxyView.as_view()),
     # tokens
     re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
     re_path(
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index c69a402ab..73d01c839 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1263,7 +1263,7 @@ def metric(self, request, **kwargs):
         )
 
 
-class ProxyMetricsView(View):
+class MetricsProxyView(View):
     cache = {}
     match_meta = staticmethod(
         re.compile(r'^(?:# (?:HELP|TYPE) )([a-zA-Z_][a-zA-Z0-9_:.-]*)').match)
@@ -1333,7 +1333,7 @@ async def stream_response():
 
 
 @method_decorator(csrf_exempt, name='dispatch')
-class PrometheusProxy(View):
+class PrometheusProxyView(View):
     timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
     authentication = authentication.DryccAuthentication()
 
@@ -1356,3 +1356,4 @@ async def proxy(self, request, username, path):
         return JsonResponse(data, status=status)
 
     get = post = proxy
+ 
\ No newline at end of file

From d387efb3dc2cd64bd921577c6270e91ed036ea40 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 2 Jul 2025 16:43:24 +0800
Subject: [PATCH 145/200] fix(measure): push networks loadbalancers metrics
 error (#185)

---
 .../api/management/commands/measure_loadbalancers.py   |  5 +++--
 rootfs/api/management/commands/measure_networks.py     | 10 ++++++----
 rootfs/api/views.py                                    |  1 -
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/rootfs/api/management/commands/measure_loadbalancers.py b/rootfs/api/management/commands/measure_loadbalancers.py
index 9997ca3ba..82fb070ca 100644
--- a/rootfs/api/management/commands/measure_loadbalancers.py
+++ b/rootfs/api/management/commands/measure_loadbalancers.py
@@ -25,8 +25,9 @@ def _measure_loadbalancers(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         loadbalancers = []
-        for metric, (_, value) in async_to_sync(monitor.query_loadbalancer(
-                app_map.keys(), start, stop)):
+        for item in async_to_sync(monitor.query_loadbalancer)(app_map.keys(), start, stop):  # noqa
+            metric = item["metric"]
+            _, value = item["value"]
             ip = metric["ip"]
             namespace = metric["namespace"]
             owner_id = app_map[namespace].owner_id
diff --git a/rootfs/api/management/commands/measure_networks.py b/rootfs/api/management/commands/measure_networks.py
index 013577d77..0dbd43967 100644
--- a/rootfs/api/management/commands/measure_networks.py
+++ b/rootfs/api/management/commands/measure_networks.py
@@ -19,8 +19,9 @@ def _measure_networks(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         networks = []
-        for metric, (_, value) in async_to_sync(monitor.query_network_receive_flow(
-                app_map.keys(), start, stop)):
+        for item in async_to_sync(monitor.query_network_receive_flow)(app_map.keys(), start, stop):  # noqa
+            metric = item["metric"]
+            _, value = item["value"]
             networks.append({
                 "app_id":  str(app_map[metric['namespace']].uuid),
                 "owner": app_map[metric['namespace']].owner_id,
@@ -33,8 +34,9 @@ def _measure_networks(self, app_map, timestamp):
                 },
                 "timestamp": start
             })
-        for metric, (_, value) in async_to_sync(monitor.query_network_transmit_flow(
-                app_map.keys(), start, stop)):
+        for item in async_to_sync(monitor.query_network_transmit_flow)(app_map.keys(), start, stop):  # noqa
+            metric = item["metric"]
+            _, value = item["value"]
             networks.append({
                 "app_id":  str(app_map[metric['namespace']].uuid),
                 "owner": app_map[metric['namespace']].owner_id,
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 73d01c839..303527ae1 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1356,4 +1356,3 @@ async def proxy(self, request, username, path):
         return JsonResponse(data, status=status)
 
     get = post = proxy
- 
\ No newline at end of file

From 5e67774d5e72c0a2e3abd75ba093227f6ed65ff4 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 10 Jul 2025 16:22:49 +0800
Subject: [PATCH 146/200] chore(quickwit): add quickwit proxy

---
 charts/controller/templates/_helpers.tpl      | 22 ++---
 .../templates/controller-secret-creds.yaml    |  1 +
 charts/controller/values.yaml                 |  3 +
 rootfs/api/authentication.py                  | 21 +++--
 rootfs/api/consumers.py                       | 42 ---------
 rootfs/api/models/app.py                      | 14 ---
 rootfs/api/models/base.py                     | 17 +++-
 rootfs/api/permissions.py                     | 10 +--
 rootfs/api/routing.py                         |  3 -
 rootfs/api/settings/production.py             | 24 ++++--
 rootfs/api/tests/test_hooks.py                | 36 ++++----
 rootfs/api/urls.py                            |  4 +
 rootfs/api/views.py                           | 86 +++++++++++++++++--
 rootfs/scheduler/__init__.py                  |  5 +-
 rootfs/scheduler/resources/deployment.py      | 22 +++--
 rootfs/scheduler/resources/hpa.py             |  5 +-
 rootfs/scheduler/resources/pv.py              |  3 +-
 rootfs/scheduler/resources/pvc.py             |  3 +-
 rootfs/scheduler/resources/rc.py              |  8 +-
 rootfs/scheduler/resources/rs.py              |  5 +-
 rootfs/scheduler/resources/scale.py           |  3 +-
 21 files changed, 194 insertions(+), 143 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 7cc23cb60..22ee25094 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -51,11 +51,11 @@ env:
     secretKeyRef:
       name: controller-creds
       key: django-secret-key
-- name: DRYCC_BUILDER_KEY
+- name: DRYCC_SERVICE_KEY
   valueFrom:
     secretKeyRef:
-      name: builder-key-auth
-      key: builder-key
+      name: controller-creds
+      key: service-key
 {{- if (.Values.valkeyUrl) }}
 - name: DRYCC_VALKEY_URL
   valueFrom:
@@ -119,18 +119,8 @@ env:
       name: controller-creds
       key: victoriametrics-url
 {{- else if .Values.victoriametrics.enabled }}
-- name: "DRYCC_VICTORIAMETRICS_USERNAME"
-  valueFrom:
-    secretKeyRef:
-      name: victoriametrics-vmauth-creds
-      key: username
-- name: "DRYCC_VICTORIAMETRICS_PASSWORD"
-  valueFrom:
-    secretKeyRef:
-      name: victoriametrics-vmauth-creds
-      key: password
 - name: "DRYCC_VICTORIAMETRICS_URL"
-  value: "http://$(DRYCC_VICTORIAMETRICS_USERNAME):$(DRYCC_VICTORIAMETRICS_PASSWORD)@drycc-victoriametrics-vmauth.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8427"
+  value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
@@ -166,6 +156,10 @@ env:
       name: controller-creds
       key: passport-secret
 {{- end }}
+- name: QUICKWIT_SEARCHER_URL
+  value: http://drycc-quickwit-searcher.{{ $.Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:7280
+- name: QUICKWIT_LOG_INDEX_PREFIX
+  value: {{ .Values.quickwit.logIndexPrefix }}
 {{- range $key, $value := .Values.environment }}
 - name: {{ $key }}
   value: {{ $value | quote }}
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index 7afbc8192..41723ce98 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -31,5 +31,6 @@ data:
   registry-username: {{ .Values.registryUsername | b64enc }}
   registry-password: {{ .Values.registryPassword | b64enc }}
   {{- end }}
+  service-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "service-key" "defaultValue" (randAscii 64) "context" $)) }}
   django-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "django-secret-key" "defaultValue" (randAscii 64) "context" $)) }}
   deploy-hook-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "deploy-hook-secret-key" "defaultValue" (randAscii 64) "context" $)) }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 7cb1d6132..a8650fea5 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -163,6 +163,9 @@ valkey:
 database:
   enabled: true
 
+quickwit:
+  logIndexPrefix: logs-
+
 registry:
   enabled: true
 
diff --git a/rootfs/api/authentication.py b/rootfs/api/authentication.py
index 34d7b0ce7..5e40f0380 100644
--- a/rootfs/api/authentication.py
+++ b/rootfs/api/authentication.py
@@ -21,6 +21,7 @@ def authenticate(self, request):
 class DryccAuthentication(authentication.BaseAuthentication):
 
     keywords = ('token', 'bearer')
+    ignore_authentication_failed = False
 
     def parse_header(self, request):
         try:
@@ -44,17 +45,19 @@ def authenticate(self, request):
         token_type, token = self.parse_header(request)
         if token_type is None or token is None:
             return None
-        if token_type == 'bearer':  # drycc oauth access token
-            try:
+        try:
+            if token_type == 'bearer':  # drycc oauth access token
                 from api.backend import OauthCacheManager
                 return OauthCacheManager().get_user(token), token
-            except exceptions.AuthenticationFailed:
-                return None
-        # drycc token
-        user = cache.get(token, None)
-        if not user:
-            return self.authenticate_credentials(token)
-        return user, token
+            # drycc token
+            user = cache.get(token, None)
+            if not user:
+                return self.authenticate_credentials(token)
+            return user, token
+        except exceptions.AuthenticationFailed as e:
+            if not self.ignore_authentication_failed:
+                raise e
+        return None
 
     def authenticate_credentials(self, key):
         from api.models.base import Token
diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index da36f4b05..1ff7450f9 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -2,7 +2,6 @@
 import time
 import six
 import ssl
-import aiohttp
 import asyncio
 from django.conf import settings
 from django.core.cache import cache
@@ -66,47 +65,6 @@ def kubernetes(self):
         return core_v1_api.CoreV1Api()
 
 
-class AppLogsConsumer(BaseAppConsumer):
-
-    async def connect(self):
-        await super().connect()
-        self.session = None
-        self.running = False
-        self.conneted = True
-
-    async def task(self, **kwargs):
-        lines = kwargs.get("lines", 100)
-        follow = kwargs.get("follow", False)
-        timeout = kwargs.get("timeout", 300)
-        url = "http://{}:{}/logs/{}?log_lines={}&follow={}&timeout={}".format(
-            settings.LOGGER_HOST, settings.LOGGER_PORT, self.id, lines, follow, timeout,
-        )
-        try:
-            async with aiohttp.ClientSession() as session:
-                self.session = session
-                async with session.get(url) as response:
-                    async for data in response.content.iter_any():
-                        if not self.conneted:
-                            break
-                        await self.send(text_data=data)
-        except asyncio.TimeoutError:
-            pass
-        finally:
-            await self.close(code=1000)
-
-    async def receive(self, text_data=None, bytes_data=None):
-        if self.running:
-            return
-        self.running = True
-        kwargs = json.loads(text_data)
-        asyncio.create_task(self.task(**kwargs))
-
-    async def disconnect(self, close_code):
-        if self.session:
-            await self.session.close()
-        self.conneted = False
-
-
 class AppPodLogsConsumer(BaseK8sConsumer):
 
     async def connect(self):
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 57aec8ce6..600374e6c 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -250,8 +250,6 @@ def delete(self, *args, **kwargs):
         except KubeHTTPException:
             # it's fine if the namespace does not exist - delete app from the DB
             pass
-
-        self._clean_app_logs()
         return super(App, self).delete(*args, **kwargs)
 
     def restart(self, **kwargs):  # noqa
@@ -752,18 +750,6 @@ def __str__(self):
     def _get_deployment_name(self, ptype):
         return f"{self.id}-{ptype}"
 
-    def _clean_app_logs(self):
-        """Delete application logs stored by the logger component"""
-        try:
-            url = 'http://{}:{}/logs/{}'.format(settings.LOGGER_HOST,
-                                                settings.LOGGER_PORT, self.id)
-            requests.delete(url)
-        except Exception as e:
-            # Ignore errors deleting application logs.  An error here should not interfere with
-            # the overall success of deleting an application, but we should log it.
-            err = 'Error deleting existing application logs: {}'.format(e)
-            self.log(err, logging.WARNING)
-
     def _mount(self, user, volume, app_settings, structure=None):
         volumes = Volume.objects.filter(app=self)
         tasks = []
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 211f7ff68..0d4c0bc7c 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -1,4 +1,5 @@
 import uuid
+import time
 import string
 import random
 from datetime import timedelta
@@ -10,7 +11,7 @@
 from django.utils.translation import gettext_lazy as _
 
 
-from api.utils import validate_json, get_scheduler
+from api.utils import get_session, validate_json, get_scheduler
 
 token_manager_oauth_schema = {
     "$schema": "http://json-schema.org/schema#",
@@ -47,6 +48,20 @@ class Meta:
         """Mark :class:`AuditedModel` as abstract."""
         abstract = True
 
+    def app_log(self, app_id, msg):
+        session = get_session()
+        session.post(
+            "",
+            json={
+                "timestamp": int(time.time()),
+                "log": msg,
+                "kubernetes": {
+                    "namespace": app_id,
+                    "container_name": "drycc-controller",
+                }
+            }
+        )
+
     @property
     def scheduler(self):
         annotations = {}
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index 4f19a7213..da41a8e05 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -111,20 +111,20 @@ def has_permission(self, request, view):
         return request.method in permissions.SAFE_METHODS or request.user.is_superuser
 
 
-class HasBuilderAuth(permissions.BasePermission):
+class IsServiceToken(permissions.BasePermission):
     """
-    View permission to allow builder to perform actions
-    with a special HTTP header
+    The service token is used for internal communication between Drycc components,
+    such as the builder and Quickwit.
     """
 
     def has_permission(self, request, view):
         """
         Return `True` if permission is granted, `False` otherwise.
         """
-        auth_header = request.META.get('HTTP_X_DRYCC_BUILDER_AUTH')
+        auth_header = request.META.get('HTTP_X_DRYCC_SERVICE_KEY')
         if not auth_header:
             return False
-        return auth_header == settings.BUILDER_KEY
+        return auth_header == settings.SERVICE_KEY
 
 
 class IsWorkflowManager(permissions.BasePermission):
diff --git a/rootfs/api/routing.py b/rootfs/api/routing.py
index b793db1cc..c37159207 100644
--- a/rootfs/api/routing.py
+++ b/rootfs/api/routing.py
@@ -4,9 +4,6 @@
 from . import consumers
 
 websocket_urlpatterns = [
-    re_path(
-        r'^v2/apps/(?P<id>([\w-]*))/logs/?$',
-        consumers.AppLogsConsumer.as_asgi()),
     re_path(
         r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<pod_id>([\w-]*))/logs/?$',
         consumers.AppPodLogsConsumer.as_asgi()),
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index da82fe81c..94e9f4f73 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -4,10 +4,17 @@
 import sys
 import uuid
 import json
+import random
+import string
 import os.path
 import tempfile
 import dj_database_url
 
+
+def randstr(k):
+    return ''.join(random.choices(string.ascii_lowercase + string.digits, k=k))
+
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.join(BASE_DIR, 'apps_extra'))
 
@@ -312,10 +319,11 @@
     with open(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH) as fd:
         DRYCC_VOLUME_CLAIM_TEMPLATE = json.load(fd)
 
-# security keys and auth tokens
-random_secret = 'CHANGEME_sapm$s%upvsw5l_zuy_&29rkywd^78ff(qi*#@&*^'
-SECRET_KEY = os.environ.get('DRYCC_SECRET_KEY', random_secret)
-BUILDER_KEY = os.environ.get('DRYCC_BUILDER_KEY', random_secret)
+# Django secret key
+SECRET_KEY = os.environ.get('DRYCC_SECRET_KEY', randstr(64))
+
+# Drycc service key
+SERVICE_KEY = os.environ.get('DRYCC_SERVICE_KEY', randstr(64))
 
 # Drycc admission mutate key
 MUTATE_KEY_PATH = os.environ.get('DRYCC_MUTATE_KEY_PATH', '/etc/controller/mutate/cert/key')
@@ -387,10 +395,6 @@
 REGISTRY_LOCATION = os.environ.get('DRYCC_REGISTRY_LOCATION', 'on-cluster')
 REGISTRY_SECRET_PREFIX = os.environ.get('DRYCC_REGISTRY_SECRET_PREFIX', 'private-registry')
 
-# logger settings
-LOGGER_HOST = os.environ.get('DRYCC_LOGGER_SERVICE_HOST', '127.0.0.1')
-LOGGER_PORT = os.environ.get('DRYCC_LOGGER_SERVICE_PORT_HTTP', 80)
-
 DRYCC_DATABASE_URL = os.environ.get('DRYCC_DATABASE_URL', 'postgres://postgres:@:5432/drycc')
 DATABASES = {
     'default': dj_database_url.config(default=DRYCC_DATABASE_URL)
@@ -479,6 +483,10 @@
     }
 }
 
+# Quickwit Configuration
+QUICKWIT_SEARCHER_URL = os.environ.get('QUICKWIT_SEARCHER_URL', None)
+QUICKWIT_LOG_INDEX_PREFIX = os.environ.get('QUICKWIT_LOG_INDEX_PREFIX', None)
+
 # Workflow-manager Configuration Options
 WORKFLOW_MANAGER_URL = os.environ.get('WORKFLOW_MANAGER_URL', None)
 WORKFLOW_MANAGER_ACCESS_KEY = os.environ.get('WORKFLOW_MANAGER_ACCESS_KEY', None)
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index 0056d4a20..d528d16b2 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -110,12 +110,12 @@ def test_key_hook(self, mock_requests):
 
         # Make sure 404 is returned for a random app
         url = '/v2/hooks/keys/doesnotexist'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
         # Test app that exists
         url = '/v2/hooks/keys/{}'.format(app_id)
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data, {"autotest": [
             {'key': rsa_pub, 'fingerprint': '54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:f5'},
@@ -126,12 +126,12 @@ def test_key_hook(self, mock_requests):
 
         # Test against an app that exist but user does not
         url = '/v2/hooks/keys/{}/foooooo'.format(app_id)
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
         # Test against an app that exists and user that does
         url = '/v2/hooks/keys/{}/{}'.format(app_id, str(self.user))
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data, {"autotest": [
             {'key': rsa_pub, 'fingerprint': '54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:f5'},
@@ -143,7 +143,7 @@ def test_key_hook(self, mock_requests):
 
         # Fetch a valid ssh key
         url = '/v2/hooks/key/54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:f5'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data, {
             "username": str(self.user),
@@ -154,12 +154,12 @@ def test_key_hook(self, mock_requests):
 
         # Fetch an non-existent base64 encoded ssh key
         url = '/v2/hooks/key/54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:wooooo'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
         # Fetch an invalid (not encoded) ssh key
         url = '/v2/hooks/key/nope'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
     def test_build_hook(self, mock_requests):
@@ -174,9 +174,9 @@ def test_build_hook(self, mock_requests):
         self.client.credentials()
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 401, response.data)
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
@@ -195,9 +195,9 @@ def test_build_hook_slug_url(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 401, response.data)
 
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
@@ -217,9 +217,9 @@ def test_build_hook_procfile(self, mock_requests):
                 'sha': SHA,
                 'procfile': PROCFILE}
 
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
@@ -258,9 +258,9 @@ def test_build_hook_dockerfile(self, mock_requests):
                 'stack': 'container',
                 'sha': SHA,
                 'dockerfile': DOCKERFILE}
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
@@ -298,9 +298,9 @@ def test_config_hook(self, mock_requests):
         self.client.credentials()
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 401, response.data)
-        # post with the builder auth key
+        # post with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('values', response.data)
         self.assertEqual(values, response.data['values'])
@@ -327,6 +327,6 @@ def test_admin_can_hook(self, mock_requests):
                 'dockerfile': DOCKERFILE}
         url = '/v2/hooks/build'
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data['release']['version'], 2)
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 898c777e4..92b134fa0 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -248,6 +248,10 @@
     re_path(
         r'^nodes/(?P<node>[a-zA-Z0-9-]+)/proxy/metrics(?:/(?P<metrics>[^/]+))?/?$',
         views.MetricsProxyView.as_view()),
+    # quickwit
+    re_path(
+        r'^quickwit/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$',
+        views.QuickwitProxyView.as_view()),
     # prometheus
     re_path(
         r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$',
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 303527ae1..0410c8b60 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -718,7 +718,7 @@ def events(self, request, **kwargs):
 
 
 class BaseHookViewSet(BaseDryccViewSet):
-    permission_classes = [permissions.HasBuilderAuth]
+    permission_classes = [permissions.IsServiceToken]
 
 
 class KeyHookViewSet(BaseHookViewSet):
@@ -1332,19 +1332,91 @@ async def stream_response():
         return StreamingHttpResponse(stream_response(), content_type=content_type)
 
 
+@method_decorator(csrf_exempt, name='dispatch')
+class QuickwitProxyView(View):
+    timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
+    permission = permissions.IsServiceToken()
+    authentication = authentication.DryccAuthentication()
+    authentication.ignore_authentication_failed = True
+    index_url_match = re.compile("^indexes/?$").match
+    search_url_match = re.compile("^(?P<index>[a-zA-Z*][\w.*-，]{0,})/search/?$").match
+
+    async def get(self, request, username, path):
+        if match := self.search_url_match(path):
+            func, index = self.query, match.group("index")
+        elif self.index_url_match(path):
+            func, index = self.index, request.GET.get("index_id_patterns", "*")
+        else:
+            raise JsonResponse({'error': 'Not Found'}, status=404)
+        if self.permission.has_permission(request, None):
+            if username != "drycc":
+                return JsonResponse({'error': 'Access denied'}, status=403)
+        else:
+            auth = await database_sync_to_async(self.authentication.authenticate)(request)
+            if not auth or len(auth) != 2:
+                return JsonResponse({'error': 'Unauthorized'}, status=401)
+            if auth[0].username != username:
+                return JsonResponse({'error': 'Access denied'}, status=403)
+            else:
+                index = await self.get_app_indexes(username, index)
+        return await func(request, index)
+
+    async def index(self, request, index):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        url, params = urljoin(base_url, f"/api/v1/indexes"), dict(request.GET)
+        params["index_id_patterns"] = index
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status, safe=False)
+
+    async def query(self, request, index):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        url, params = urljoin(base_url, f"/api/v1/{index}/search"), dict(request.GET)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    async def get_app_indexes(self, username, index):
+        if "," in index:
+            match = re.compile("|".join([f"^{i}$" for i in index.split(",")])).match
+        else:
+            match = re.compile(index).match
+        app_indexes, log_index_prefix = [], settings.QUICKWIT_LOG_INDEX_PREFIX
+        async for app in models.app.App.objects.filter(owner__username=username):
+            app_index = f"{log_index_prefix}{app.id}"
+            if match(app_index):
+                app_indexes.append(app_index)
+        return ",".join(app_indexes)
+
+
 @method_decorator(csrf_exempt, name='dispatch')
 class PrometheusProxyView(View):
     timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
+    permission = permissions.IsServiceToken()
     authentication = authentication.DryccAuthentication()
+    authentication.ignore_authentication_failed = True
 
     async def proxy(self, request, username, path):
-        auth = await database_sync_to_async(self.authentication.authenticate)(request)
-        if not auth or len(auth) != 2:
-            return JsonResponse({'error': 'Unauthorized'}, status=401)
-        if auth[0].username != username:
-            return JsonResponse({'error': 'Access denied'}, status=403)
+        if self.permission.has_permission(request, None):
+            if username != "drycc":
+                return JsonResponse({'error': 'Access denied'}, status=403)
+            path = f"/select/0/prometheus/{path}"
         else:
-            path = f"/select/{auth[0].id}/prometheus/{path}"
+            auth = await database_sync_to_async(self.authentication.authenticate)(request)
+            if not auth or len(auth) != 2:
+                return JsonResponse({'error': 'Unauthorized'}, status=401)
+            if auth[0].username != username:
+                return JsonResponse({'error': 'Access denied'}, status=403)
+            else:
+                path = f"/select/{auth[0].id}/prometheus/{path}"
         url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, path)
         params = dict(request.GET) if request.method == "GET" else dict(request.POST)
         try:
diff --git a/rootfs/scheduler/__init__.py b/rootfs/scheduler/__init__.py
index 94c51c6f7..72d3c0376 100644
--- a/rootfs/scheduler/__init__.py
+++ b/rootfs/scheduler/__init__.py
@@ -141,7 +141,7 @@ def query_params(labels=None, fields=None, resource_version=None, pretty=False):
         return query
 
     @staticmethod
-    def log(namespace, message, level='INFO'):
+    def log(namespace, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
         This prefixes log messages with a namespace "tag".
@@ -149,8 +149,7 @@ def log(namespace, message, level='INFO'):
         sort like releasing or scaling, will be considered as "belonging" to the application
         instead of the controller and will be handled accordingly.
         """
-        lvl = getattr(logging, level.upper()) if hasattr(logging, level.upper()) else logging.INFO
-        logger.log(lvl, "[{}]: {}".format(namespace, message))
+        logger.log(level, "[{}]: {}".format(namespace, message))
 
     def http_head(self, path, **kwargs):
         """
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index 1f9eeef49..bf9485553 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -1,6 +1,7 @@
-from datetime import datetime, timedelta, timezone
 import json
 import time
+import logging
+from datetime import datetime, timedelta, timezone
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeException, KubeHTTPException
 
@@ -121,7 +122,8 @@ def create(self, namespace, name, image, command, args,
         url = self.api("/namespaces/{}/deployments", namespace)
         response = self.http_post(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
             if not ignore_exception:
                 raise KubeHTTPException(
                     response,
@@ -140,7 +142,8 @@ def update(self, namespace, name, image, command, args,
         url = self.api("/namespaces/{}/deployments/{}", namespace, name)
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
             if not ignore_exception:
                 raise KubeHTTPException(response, 'update Deployment "{}"', name)
         else:
@@ -161,7 +164,8 @@ def patch(self, namespace, name, image, command, args,
         )
 
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
             if not ignore_exception:
                 raise KubeHTTPException(response, 'patch Deployment "{}"', name)
         else:
@@ -245,17 +249,17 @@ def in_progress(self, namespace, name, timeout, batches, replicas, tags):
         Returns 2 booleans, first one is for if the Deployment is in progress or not, second
         one is or if a rollback action is advised while leaving the rollback up to the caller
         """
-        self.log(namespace, 'Checking if Deployment {} is in progress'.format(name), level='DEBUG')  # noqa
+        self.log(namespace, 'Checking if Deployment {} is in progress'.format(name), level=logging.DEBUG)  # noqa
         try:
             ready, _ = self.are_replicas_ready(namespace, name)
             if ready:
                 # nothing more to do - False since it is not in progress
-                self.log(namespace, 'All replicas for Deployment {} are ready'.format(name), level='DEBUG')  # noqa
+                self.log(namespace, 'All replicas for Deployment {} are ready'.format(name), level=logging.DEBUG)  # noqa
                 return False, False
         except KubeHTTPException as e:
             # Deployment doesn't exist
             if e.response.status_code == 404:
-                self.log(namespace, 'Deployment {} does not exist yet'.format(name), level='DEBUG')  # noqa
+                self.log(namespace, 'Deployment {} does not exist yet'.format(name), level=logging.DEBUG)  # noqa
                 return False, False
 
         # get deployment information
@@ -348,7 +352,7 @@ def wait_until_updated(self, namespace, name):
         More information is also available at:
         https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
-        self.log(namespace, "waiting for Deployment {} to get a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+        self.log(namespace, "waiting for Deployment {} to get a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
         for _ in range(30):
             try:
                 deploy = self.deployment.get(namespace, name).json()
@@ -356,7 +360,7 @@ def wait_until_updated(self, namespace, name):
                     'observedGeneration' in deploy['status'] and
                     deploy['status']['observedGeneration'] >= deploy['metadata']['generation']
                 ):
-                    self.log(namespace, "A newer generation was found for Deployment {}".format(name), 'DEBUG')  # noqa
+                    self.log(namespace, "A newer generation was found for Deployment {}".format(name), logging.DEBUG)  # noqa
                     break
 
                 time.sleep(1)
diff --git a/rootfs/scheduler/resources/hpa.py b/rootfs/scheduler/resources/hpa.py
index f45d810f8..6523a0adf 100644
--- a/rootfs/scheduler/resources/hpa.py
+++ b/rootfs/scheduler/resources/hpa.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from packaging.version import parse
 
 from scheduler.resources import Resource
@@ -93,7 +94,7 @@ def create(self, namespace, name, app_type, target, **kwargs):
         url = self.api("/namespaces/{}/horizontalpodautoscalers", namespace)
         response = self.http_post(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(
                 response,
                 'create HorizontalPodAutoscaler "{}" in Namespace "{}"', name, namespace
@@ -111,7 +112,7 @@ def update(self, namespace, name, app_type, target, **kwargs):
         url = self.api("/namespaces/{}/horizontalpodautoscalers/{}", namespace, name)
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(response, 'update HorizontalPodAutoscaler "{}"', name)
 
         # optionally wait for HPA if requested
diff --git a/rootfs/scheduler/resources/pv.py b/rootfs/scheduler/resources/pv.py
index afb8d8b10..947d1c490 100644
--- a/rootfs/scheduler/resources/pv.py
+++ b/rootfs/scheduler/resources/pv.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -57,7 +58,7 @@ def patch(self, name, **kwargs):
         data = self.manifest(name, **kwargs)
         response = self.http_patch(url, json=data, headers={"Content-Type": "application/merge-patch+json"})  # noqa
         if self.unhealthy(response.status_code):
-            self.log('template used: {}'.format(json.dumps(data, indent=4)), 'DEBUG')
+            self.log('template used: {}'.format(json.dumps(data, indent=4)), logging.DEBUG)
             raise KubeHTTPException(response, 'update persistentvolume "{}"', name)
         return response
 
diff --git a/rootfs/scheduler/resources/pvc.py b/rootfs/scheduler/resources/pvc.py
index 5ce7df202..df5aa7b40 100644
--- a/rootfs/scheduler/resources/pvc.py
+++ b/rootfs/scheduler/resources/pvc.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -59,7 +60,7 @@ def patch(self, namespace, name, **kwargs):
         data = self.manifest(namespace, name, **kwargs)
         response = self.http_patch(url, json=data, headers={"Content-Type": "application/merge-patch+json"})  # noqa
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(data, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(data, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(response, 'update persistentvolumeclaims "{}"', name)
         return response
 
diff --git a/rootfs/scheduler/resources/rc.py b/rootfs/scheduler/resources/rc.py
index cfdf9f12f..f5d1fce56 100644
--- a/rootfs/scheduler/resources/rc.py
+++ b/rootfs/scheduler/resources/rc.py
@@ -1,5 +1,6 @@
 import json
 import time
+import logging
 from scheduler.exceptions import KubeHTTPException
 from scheduler.resources import Resource
 
@@ -56,7 +57,8 @@ def create(self, namespace, name, image, command, args, **kwargs):
         url = self.api("/namespaces/{}/replicationcontrollers", namespace)
         resp = self.http_post(url, json=manifest)
         if self.unhealthy(resp.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
             raise KubeHTTPException(
                 resp,
                 'create ReplicationController "{}" in Namespace "{}"', name, namespace
@@ -114,7 +116,7 @@ def wait_until_updated(self, namespace, name):
         More information is also available at:
         https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
-        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
         for _ in range(30):
             try:
                 rc = self.get(namespace, name).json()
@@ -122,7 +124,7 @@ def wait_until_updated(self, namespace, name):
                     "observedGeneration" in rc["status"] and
                     rc["status"]["observedGeneration"] >= rc["metadata"]["generation"]
                 ):
-                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
                     break
 
                 time.sleep(1)
diff --git a/rootfs/scheduler/resources/rs.py b/rootfs/scheduler/resources/rs.py
index bde61d188..289c26949 100644
--- a/rootfs/scheduler/resources/rs.py
+++ b/rootfs/scheduler/resources/rs.py
@@ -1,4 +1,5 @@
 import time
+import logging
 from scheduler.exceptions import KubeHTTPException
 from scheduler.resources import Resource
 
@@ -69,7 +70,7 @@ def wait_until_updated(self, namespace, name):
         More information is also available at:
         https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
-        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
         for _ in range(30):
             try:
                 rs = self.get(namespace, name).json()
@@ -77,7 +78,7 @@ def wait_until_updated(self, namespace, name):
                     "observedGeneration" in rs["status"] and
                     rs["status"]["observedGeneration"] >= rs["metadata"]["generation"]
                 ):
-                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
                     break
 
                 time.sleep(1)
diff --git a/rootfs/scheduler/resources/scale.py b/rootfs/scheduler/resources/scale.py
index 43f72eb4c..fd5babcec 100644
--- a/rootfs/scheduler/resources/scale.py
+++ b/rootfs/scheduler/resources/scale.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -32,7 +33,7 @@ def update(self, namespace, name, replicas, target):
         url = self.api("/namespaces/{}/{}/{}/scale", namespace, resource_type, name)
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(
                 response,
                 'scale {} "{}" in Namespace "{}"', target['kind'], name, namespace

From 9d5f97fb25663a01b1d330c8b0af56e25187d012 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 15 Jul 2025 23:09:55 +0800
Subject: [PATCH 147/200] chore(controller): remove cluster domain

---
 charts/controller/templates/_helpers.tpl             | 12 +++++-------
 .../templates/controller-celery-deloyment.yaml       |  4 +---
 .../templates/controller-cronjob-daily.yaml          |  4 +---
 .../templates/controller-cronjob-hourly.yaml         |  4 ++--
 .../templates/controller-mutate-deloyment.yaml       |  4 +---
 charts/controller/values.yaml                        |  5 -----
 rootfs/api/models/service.py                         |  5 +----
 rootfs/api/settings/production.py                    | 11 ++---------
 rootfs/api/tests/test_app.py                         |  9 +++------
 rootfs/api/tests/test_service.py                     | 10 +++-------
 rootfs/api/views.py                                  |  6 +++---
 11 files changed, 22 insertions(+), 52 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 22ee25094..1941e0df7 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -17,8 +17,6 @@ env:
   value: "{{ (tpl .Values.filerImage .) }}"
 - name: "DRYCC_FILER_IMAGE_PULL_POLICY"
   value: "{{ (tpl .Values.filerImagePullPolicy .) }}"
-- name: "KUBERNETES_CLUSTER_DOMAIN"
-  value: "{{ .Values.global.clusterDomain }}"
 - name: "DRYCC_APP_GATEWAY_CLASS"
   value: "{{ .Values.appGatewayClass }}"
 {{- if (.Values.appStorageClass) }}
@@ -69,7 +67,7 @@ env:
       name: valkey-creds
       key: password
 - name: DRYCC_VALKEY_URL
-  value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:16379/0"
+  value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey:16379/0"
 {{- end }}
 {{- if (.Values.databaseReplicaUrl) }}
 - name: DRYCC_DATABASE_REPLICA_URL
@@ -96,9 +94,9 @@ env:
       name: database-creds
       key: password
 - name: DRYCC_DATABASE_URL
-  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/controller"
+  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database:5432/controller"
 - name: DRYCC_DATABASE_REPLICA_URL
-  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/controller"
+  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database-replica:5432/controller"
 {{- end }}
 {{- if (.Values.workflowManagerUrl) }}
 - name: WORKFLOW_MANAGER_URL
@@ -120,7 +118,7 @@ env:
       key: victoriametrics-url
 {{- else if .Values.victoriametrics.enabled }}
 - name: "DRYCC_VICTORIAMETRICS_URL"
-  value: "http://drycc-victoriametrics-vmselect.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:8481"
+  value: "http://drycc-victoriametrics-vmselect:8481"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
@@ -157,7 +155,7 @@ env:
       key: passport-secret
 {{- end }}
 - name: QUICKWIT_SEARCHER_URL
-  value: http://drycc-quickwit-searcher.{{ $.Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:7280
+  value: http://drycc-quickwit-searcher:7280
 - name: QUICKWIT_LOG_INDEX_PREFIX
   value: {{ .Values.quickwit.logIndexPrefix }}
 {{- range $key, $value := .Values.environment }}
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/controller-celery-deloyment.yaml
index c11f26b61..186353088 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/controller-celery-deloyment.yaml
@@ -34,9 +34,7 @@ spec:
           - netcat
           - -v
           - -u
-          - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
-          - -a
-          - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+          - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller-celery
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index 5766e0290..3e68cc4e8 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -29,9 +29,7 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
-              - -a
-              - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
             {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 0760503da..15987d248 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -30,8 +30,8 @@ spec:
               - -v
               - -u
               - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
-              - -a
-              - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+              - -u
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
             {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
diff --git a/charts/controller/templates/controller-mutate-deloyment.yaml b/charts/controller/templates/controller-mutate-deloyment.yaml
index 4f20a0e74..fbbda5d93 100644
--- a/charts/controller/templates/controller-mutate-deloyment.yaml
+++ b/charts/controller/templates/controller-mutate-deloyment.yaml
@@ -24,9 +24,7 @@ spec:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
-        - -a
-        - $(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
         {{- include "controller.envs" . | indent 8 }}
       containers:
       - name: drycc-controller
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index a8650fea5..b37f0d4d0 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -178,11 +178,6 @@ victoriametrics:
 global:
   # Admin email, used for each component to send email to administrator
   email: "drycc@drycc.cc"
-  # A domain name consists of one or more parts.
-  # Periods (.) are used to separate these parts.
-  # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-).
-  # It must start and end with a lowercase letter or digit.
-  clusterDomain: "cluster.local"
   # The public resolvable hostname to build your cluster with.
   #
   # This will be the hostname that is used to build endpoints such as "drycc.$HOSTNAME"
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index 47a46dc17..f5222838b 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -1,7 +1,6 @@
 import logging
 from functools import partial
 from django.db import models
-from django.conf import settings
 from django.contrib.auth import get_user_model
 from api.utils import validate_json
 from api.exceptions import ServiceUnavailable
@@ -53,9 +52,7 @@ def name(self):
 
     @property
     def domain(self):
-        return "{}.{}.svc.{}".format(
-            self.name, self.namespace, settings.KUBERNETES_CLUSTER_DOMAIN
-        )
+        return "{}.{}.svc".format(self.name, self.namespace)
 
     @property
     def namespace(self):
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 94e9f4f73..7b5a542e4 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -266,18 +266,11 @@ def randstr(k):
 # the k8s namespace in which the controller and workflow were installed.
 WORKFLOW_NAMESPACE = os.environ.get('WORKFLOW_NAMESPACE', 'drycc')
 
-# kubernetes cluster domain
-KUBERNETES_CLUSTER_DOMAIN = os.environ.get("KUBERNETES_CLUSTER_DOMAIN", "cluster.local")
 # default scheduler settings
 SCHEDULER_MODULE = 'scheduler'
 SCHEDULER_URL = "https://{}:{}".format(
-    # accessing the k8s api server by IP address rather than hostname avoids
-    # intermittent DNS errors
-    os.environ.get(
-        'KUBERNETES_SERVICE_HOST',
-        'kubernetes.default.svc.{}'.format(KUBERNETES_CLUSTER_DOMAIN)
-    ),
-    os.environ.get('KUBERNETES_SERVICE_PORT', '443')
+    os.environ.get('KUBERNETES_SERVICE_HOST', 'kubernetes.default'),
+    os.environ.get('KUBERNETES_SERVICE_PORT', '443'),
 )
 
 K8S_API_VERIFY_TLS = os.environ.get('K8S_API_VERIFY_TLS', 'true').lower() == "true"
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index e36e1d370..02895f794 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -452,8 +452,7 @@ def test_app_verify_application_health_success(self, mock_requests):
 
         # create app
         app_id = self.create_app()
-        hostname = 'http://{}.{}.svc.{}:80/'.format(
-            app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
+        hostname = 'http://{}.{}.svc:80/'.format(app_id, app_id)
         mr = mock_requests.register_uri('GET', hostname, responses)
 
         # deploy app to get verification
@@ -488,8 +487,7 @@ def test_app_verify_application_health_failure_404(self, mock_requests):
         # create app
         app_id = self.create_app()
 
-        hostname = 'http://{}.{}.svc.{}:80/'.format(
-            app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
+        hostname = 'http://{}.{}.svc:80/'.format(app_id, app_id)
         mr = mock_requests.register_uri('GET', hostname, responses)
         # deploy app to get verification
         url = "/v2/apps/{}/build".format(app_id)
@@ -512,8 +510,7 @@ def _raise_exception(request, ctx):
         # create app
         app_id = self.create_app()
         # function tries to hit router 10 times
-        hostname = 'http://{}.{}.svc.{}:80/'.format(
-            app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
+        hostname = 'http://{}.{}.svc:80/'.format(app_id, app_id)
         mr = mock_requests.register_uri('GET', hostname, text=_raise_exception)
 
         # deploy app to get verification
diff --git a/rootfs/api/tests/test_service.py b/rootfs/api/tests/test_service.py
index 869c2c201..85f8f4b3a 100644
--- a/rootfs/api/tests/test_service.py
+++ b/rootfs/api/tests/test_service.py
@@ -1,6 +1,5 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
 
 from api.tests import DryccTransactionTestCase
 from api.tests.test_gateway import RouteTest
@@ -50,8 +49,7 @@ def test_service_basic_ops(self):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['services']), 1)
         expected0 = {
-            "domain": "%s-%s.%s.svc.%s" % (
-                app_id, "test", app_id, settings.KUBERNETES_CLUSTER_DOMAIN),
+            "domain": "%s-%s.%s.svc" % (app_id, "test", app_id),
             "ports": [{
                 'name': "%s-%s-%s-%s" % (app_id, "test", 'udp', 5000),
                 'port': 5000,
@@ -75,8 +73,7 @@ def test_service_basic_ops(self):
 
         # add new port
         expected1 = {
-            "domain": "%s-%s.%s.svc.%s" % (
-                app_id, "test", app_id, settings.KUBERNETES_CLUSTER_DOMAIN),
+            "domain": "%s-%s.%s.svc" % (app_id, "test", app_id),
             "ports": [
                 {
                     'name': "%s-%s-%s-%s" % (app_id, "test", 'udp', 5000),
@@ -124,8 +121,7 @@ def test_service_basic_ops(self):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['services']), 2)
         expected2 = {
-            "domain": "%s-%s.%s.svc.%s" % (
-                app_id, "test2", app_id, settings.KUBERNETES_CLUSTER_DOMAIN),
+            "domain": "%s-%s.%s.svc" % (app_id, "test2", app_id),
             "ports": [{
                 'name': "%s-%s-%s-%s" % (app_id, "test2", 'udp', 5000),
                 'port': 5000,
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 0410c8b60..b645e5ef2 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1338,8 +1338,8 @@ class QuickwitProxyView(View):
     permission = permissions.IsServiceToken()
     authentication = authentication.DryccAuthentication()
     authentication.ignore_authentication_failed = True
-    index_url_match = re.compile("^indexes/?$").match
-    search_url_match = re.compile("^(?P<index>[a-zA-Z*][\w.*-，]{0,})/search/?$").match
+    index_url_match = re.compile(r"^indexes/?$").match
+    search_url_match = re.compile(r"^(?P<index>[a-zA-Z*][\w.*-，]{0,})/search/?$").match
 
     async def get(self, request, username, path):
         if match := self.search_url_match(path):
@@ -1363,7 +1363,7 @@ async def get(self, request, username, path):
 
     async def index(self, request, index):
         base_url = settings.QUICKWIT_SEARCHER_URL
-        url, params = urljoin(base_url, f"/api/v1/indexes"), dict(request.GET)
+        url, params = urljoin(base_url, "/api/v1/indexes"), dict(request.GET)
         params["index_id_patterns"] = index
         try:
             async with aiohttp.ClientSession() as session:

From 594d0fee1ab315dbdef435aaf6bac442ccb07cb6 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 15 Jul 2025 23:10:24 +0800
Subject: [PATCH 148/200] chore(deps): bump aiohttp from 3.11.16 to 3.12.14 in
 /rootfs (#186)

Bumps [aiohttp](https://github.com/aio-libs/aiohttp) from 3.11.16 to 3.12.14.
- [Release notes](https://github.com/aio-libs/aiohttp/releases)
- [Changelog](https://github.com/aio-libs/aiohttp/blob/master/CHANGES.rst)
- [Commits](https://github.com/aio-libs/aiohttp/compare/v3.11.16...v3.12.14)

---
updated-dependencies:
- dependency-name: aiohttp
  dependency-version: 3.12.14
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index fc0eaf837..d5ea6aa1b 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -3,7 +3,7 @@ inotify==0.2.10
 backoff==2.2.1
 django==4.2.22
 channels==4.2.2
-aiohttp==3.11.16
+aiohttp==3.12.14
 django-cors-headers==4.7.0
 django-guardian==2.4.0
 djangorestframework==3.16.0

From 02d90f2012ca4b7200b95546b392dd85ee3a9127 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 16 Jul 2025 20:54:05 +0800
Subject: [PATCH 149/200] feat(quickwit): add opensearch compatible API

---
 rootfs/api/views.py | 77 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 65 insertions(+), 12 deletions(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index b645e5ef2..d3db8f15a 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1340,14 +1340,22 @@ class QuickwitProxyView(View):
     authentication.ignore_authentication_failed = True
     index_url_match = re.compile(r"^indexes/?$").match
     search_url_match = re.compile(r"^(?P<index>[a-zA-Z*][\w.*-，]{0,})/search/?$").match
+    msearch_url_match = re.compile(r"^_elastic/_msearch/?$").match
+    field_caps_url_match = re.compile(r"_elastic/(?P<index>[a-zA-Z*][\w.*-，]{0,})/_field_caps/?$").match
 
-    async def get(self, request, username, path):
-        if match := self.search_url_match(path):
-            func, index = self.query, match.group("index")
-        elif self.index_url_match(path):
-            func, index = self.index, request.GET.get("index_id_patterns", "*")
+    async def proxy(self, request, username, path):
+        kwargs = {"request": request, "username": username}
+        if self.index_url_match(path):
+            func, kwargs["index"] = self.index, request.GET.get("index_id_patterns", "*")
+        elif match := self.search_url_match(path):
+            func, kwargs["index"] = self.query, match.group("index")
+        elif self.msearch_url_match(path):
+            func = self.msearch
+        elif match := self.field_caps_url_match(path):
+            func, kwargs["index"] = self.field_caps, match.group("index")
         else:
             raise JsonResponse({'error': 'Not Found'}, status=404)
+
         if self.permission.has_permission(request, None):
             if username != "drycc":
                 return JsonResponse({'error': 'Access denied'}, status=403)
@@ -1357,12 +1365,11 @@ async def get(self, request, username, path):
                 return JsonResponse({'error': 'Unauthorized'}, status=401)
             if auth[0].username != username:
                 return JsonResponse({'error': 'Access denied'}, status=403)
-            else:
-                index = await self.get_app_indexes(username, index)
-        return await func(request, index)
+        return await func(**kwargs)
 
-    async def index(self, request, index):
+    async def index(self, request, username, index):
         base_url = settings.QUICKWIT_SEARCHER_URL
+        index = await self.get_app_indexes(username, index)
         url, params = urljoin(base_url, "/api/v1/indexes"), dict(request.GET)
         params["index_id_patterns"] = index
         try:
@@ -1373,8 +1380,9 @@ async def index(self, request, index):
             data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
         return JsonResponse(data, status=status, safe=False)
 
-    async def query(self, request, index):
+    async def query(self, request, username, index):
         base_url = settings.QUICKWIT_SEARCHER_URL
+        index = await self.get_app_indexes(username, index)
         url, params = urljoin(base_url, f"/api/v1/{index}/search"), dict(request.GET)
         try:
             async with aiohttp.ClientSession() as session:
@@ -1384,18 +1392,63 @@ async def query(self, request, index):
             data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
         return JsonResponse(data, status=status)
 
+    async def msearch(self, request, username):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        json_lines = request.body.decode('utf-8').strip().split('\n')
+        for i, json_line in enumerate(json_lines):
+            if i % 2 == 0:
+                request_header = json.loads(json_line)
+                request_header['index'] = ",".join(
+                    [await self.get_app_indexes(username, i) for i in request_header['index']]
+                ).split(",")
+                json_lines[i] = json.dumps(request_header)
+        url, params = urljoin(
+            base_url, f"/api/v1/_elastic/_msearch"), dict(request.GET)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    url, data="\n".join(json_lines), params=params, timeout=self.timeout
+                ) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    async def field_caps(self, request, username, index):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        index = await self.get_app_indexes(username, index)
+        url, params = urljoin(
+            base_url, f"/api/v1/_elastic/{index}/_field_caps"), dict(request.GET)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
     async def get_app_indexes(self, username, index):
+        if username == "drycc":
+            return index
         if "," in index:
             match = re.compile("|".join([f"^{i}$" for i in index.split(",")])).match
         else:
             match = re.compile(index).match
         app_indexes, log_index_prefix = [], settings.QUICKWIT_LOG_INDEX_PREFIX
-        async for app in models.app.App.objects.filter(owner__username=username):
-            app_index = f"{log_index_prefix}{app.id}"
+        if hasattr(self, "app_ids"):
+            app_ids = self.app_ids
+        else:
+            app_ids = [
+                app.id async for app in models.app.App.objects.filter(owner__username=username)]
+            setattr(self, "app_ids", app_ids)
+        for app_id in app_ids:
+            app_index = f"{log_index_prefix}{app_id}"
             if match(app_index):
                 app_indexes.append(app_index)
         return ",".join(app_indexes)
 
+    get = post = proxy
+
 
 @method_decorator(csrf_exempt, name='dispatch')
 class PrometheusProxyView(View):

From 82faa52d7e995e771d1c94f8e08ebb200d3190bc Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 16 Jul 2025 22:35:31 +0800
Subject: [PATCH 150/200] chore(proxy): add base user proxy view

---
 rootfs/api/views.py | 73 +++++++++++++++++++++++++--------------------
 1 file changed, 40 insertions(+), 33 deletions(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index d3db8f15a..41fb361c9 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1333,17 +1333,41 @@ async def stream_response():
 
 
 @method_decorator(csrf_exempt, name='dispatch')
-class QuickwitProxyView(View):
+class BaseUserProxyView(View):
     timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
     permission = permissions.IsServiceToken()
     authentication = authentication.DryccAuthentication()
     authentication.ignore_authentication_failed = True
+
+    async def authenticate(self, request, username):
+        """
+        Authenticate the user based on the provided request and username.
+        Returns the user ID on success; returns None and an error message on failure.
+        """
+        if self.permission.has_permission(request, None):
+            if username != "drycc":
+                return None, {'error': 'Access denied', "status_code": 403}
+            return -1, None
+        auth = await database_sync_to_async(self.authentication.authenticate)(request)
+        if not auth or len(auth) != 2:
+            return None, {'error': 'Unauthorized', "status_code": 401}
+        if auth[0].username != username:
+            return None, {'error': 'Access denied', "status_code": 403}
+        return auth[0].id, None        
+
+
+@method_decorator(csrf_exempt, name='dispatch')
+class QuickwitProxyView(BaseUserProxyView):
     index_url_match = re.compile(r"^indexes/?$").match
     search_url_match = re.compile(r"^(?P<index>[a-zA-Z*][\w.*-，]{0,})/search/?$").match
     msearch_url_match = re.compile(r"^_elastic/_msearch/?$").match
-    field_caps_url_match = re.compile(r"_elastic/(?P<index>[a-zA-Z*][\w.*-，]{0,})/_field_caps/?$").match
+    field_caps_url_match = re.compile(
+        r"_elastic/(?P<index>[a-zA-Z*][\w.*-，]{0,})/_field_caps/?$").match
 
     async def proxy(self, request, username, path):
+        user_id, message = await self.authenticate(request, username)
+        if user_id is None and message is not None:
+            return JsonResponse(message, status=message["status_code"])
         kwargs = {"request": request, "username": username}
         if self.index_url_match(path):
             func, kwargs["index"] = self.index, request.GET.get("index_id_patterns", "*")
@@ -1354,17 +1378,7 @@ async def proxy(self, request, username, path):
         elif match := self.field_caps_url_match(path):
             func, kwargs["index"] = self.field_caps, match.group("index")
         else:
-            raise JsonResponse({'error': 'Not Found'}, status=404)
-
-        if self.permission.has_permission(request, None):
-            if username != "drycc":
-                return JsonResponse({'error': 'Access denied'}, status=403)
-        else:
-            auth = await database_sync_to_async(self.authentication.authenticate)(request)
-            if not auth or len(auth) != 2:
-                return JsonResponse({'error': 'Unauthorized'}, status=401)
-            if auth[0].username != username:
-                return JsonResponse({'error': 'Access denied'}, status=403)
+            return JsonResponse({'error': 'Not Found'}, status=404)
         return await func(**kwargs)
 
     async def index(self, request, username, index):
@@ -1403,7 +1417,7 @@ async def msearch(self, request, username):
                 ).split(",")
                 json_lines[i] = json.dumps(request_header)
         url, params = urljoin(
-            base_url, f"/api/v1/_elastic/_msearch"), dict(request.GET)
+            base_url, "/api/v1/_elastic/_msearch"), dict(request.GET)
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.post(
@@ -1451,30 +1465,23 @@ async def get_app_indexes(self, username, index):
 
 
 @method_decorator(csrf_exempt, name='dispatch')
-class PrometheusProxyView(View):
-    timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
-    permission = permissions.IsServiceToken()
-    authentication = authentication.DryccAuthentication()
-    authentication.ignore_authentication_failed = True
-
+class PrometheusProxyView(BaseUserProxyView):
     async def proxy(self, request, username, path):
-        if self.permission.has_permission(request, None):
-            if username != "drycc":
-                return JsonResponse({'error': 'Access denied'}, status=403)
+        user_id, message = await self.authenticate(request, username)
+        if user_id is None and message is not None:
+            return JsonResponse(message, status=message["status_code"])
+        if username == "drycc":
             path = f"/select/0/prometheus/{path}"
         else:
-            auth = await database_sync_to_async(self.authentication.authenticate)(request)
-            if not auth or len(auth) != 2:
-                return JsonResponse({'error': 'Unauthorized'}, status=401)
-            if auth[0].username != username:
-                return JsonResponse({'error': 'Access denied'}, status=403)
-            else:
-                path = f"/select/{auth[0].id}/prometheus/{path}"
-        url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, path)
-        params = dict(request.GET) if request.method == "GET" else dict(request.POST)
+            path = f"/select/{user_id}/prometheus/{path}"
         try:
             async with aiohttp.ClientSession() as session:
-                async with session.get(url, params=params, timeout=self.timeout) as response:
+                async with session.post(
+                    urljoin(settings.DRYCC_VICTORIAMETRICS_URL, path),
+                    data=dict(request.GET) if request.method == "GET" else dict(request.POST),
+                    headers={"Content-Type": "application/x-www-form-urlencoded"},
+                    timeout=self.timeout
+                ) as response:
                     data, status = await response.json(), response.status
         except aiohttp.ClientError as e:
             data, status = {'error': f'victoriametrics connection failed: {str(e)}'}, 502

From dc6357a96d4db2beddf62408f75ac1b6266dfe2b Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 17 Jul 2025 11:52:29 +0800
Subject: [PATCH 151/200] chore(volume): add reserved name patterns

---
 charts/controller/templates/_helpers.tpl      | 36 ++++++++-----------
 .../templates/controller-configmap.yaml       | 12 +++----
 rootfs/api/models/app.py                      | 19 +++++-----
 rootfs/api/models/gateway.py                  |  3 +-
 rootfs/api/serializers/__init__.py            |  4 ++-
 rootfs/api/settings/production.py             | 12 +++----
 rootfs/api/tests/test_app.py                  |  4 +--
 rootfs/api/utils.py                           |  8 +++++
 rootfs/api/views.py                           | 12 ++++++-
 9 files changed, 60 insertions(+), 50 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 1941e0df7..e579365bc 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -320,28 +320,20 @@ kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim, job]
 {{- end }}
 
-{{/* Generate controller config default reserved names */}}
-{{ define "controller.config.defaultReservedNames" }}
-backup
-catalog
-cert-manager
-default
-drycc
-drycc-manager
-drycc-helmbroker
-drycc-builder
-drycc-grafana
-drycc-passport
-istio-gateway
-istio-system
-kube-node-lease
-kube-public
-kube-system
-longhorn-system
-metallb
-mount-s3
-topolvm
-rook-ceph
+{{/* Generate controller config default reserved name patterns */}}
+{{ define "controller.config.defaultReservedNamePatterns" }}
+^backup$
+^catalog$
+^cert-manager$
+^default$
+^drycc(?:-[\w-]+)?$
+^istio(?:-[\w-]+)?$
+^kube(?:-[\w-]+)?$
+^longhorn-system$
+^metallb$
+^mount-s3$
+^topolvm$
+^rook-ceph$
 {{- end }}
 
 {{/* Generate controller config default secrets template */}}
diff --git a/charts/controller/templates/controller-configmap.yaml b/charts/controller/templates/controller-configmap.yaml
index 15476dcbe..2a4cdca92 100644
--- a/charts/controller/templates/controller-configmap.yaml
+++ b/charts/controller/templates/controller-configmap.yaml
@@ -23,12 +23,6 @@ data:
     {{- else}}
     {{- include "controller.config.defaultLimitPlans" . | fromYamlArray | toPrettyJson | nindent 4 }}
     {{- end }}
-  reserved-names.txt: |-
-    {{- if .Values.config.reservedNames }}
-    {{- (tpl .Values.config.reservedNames $)  | nindent 4 }}
-    {{- else}}
-    {{- include "controller.config.defaultReservedNames" . | nindent 4 }}
-    {{- end }}
   secret-template.json: |
     {{- if .Values.config.secretTemplate }}
     {{- (tpl .Values.config.secretTemplate $)  | nindent 4 }}
@@ -41,6 +35,12 @@ data:
     {{- else}}
     {{- include "controller.config.defaultVolumeTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
     {{- end }}
+  reserved-name-patterns.txt: |-
+    {{- if .Values.config.reservedNames }}
+    {{- (tpl .Values.config.reservedNames $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultReservedNamePatterns" . | nindent 4 }}
+    {{- end }}
   volume-claim-template.json: |
     {{- if .Values.config.volumeClaimTemplate }}
     {{- (tpl .Values.config.volumeClaimTemplate $)  | nindent 4 }}
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 600374e6c..194cb6df0 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -22,7 +22,8 @@
 
 from api.utils import get_session, dict_diff
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
-from api.utils import CacheLock, DeployLock, generate_app_name, apply_tasks
+from api.utils import (
+    CacheLock, DeployLock, generate_app_name, apply_tasks, validate_reserved_names)
 from scheduler import KubeHTTPException, KubeException
 from .gateway import Gateway, Route
 from .limit import LimitPlan
@@ -88,23 +89,20 @@ def validate_app_id(value):
     if not match:
         raise ValidationError("App name must start with an alphabetic character, cannot end with a"
                               + " hyphen and can only contain a-z (lowercase), 0-9 and hyphens.")
+    validate_reserved_names(value)
 
 
 def validate_app_structure(value):
     """Error if the dict values aren't ints >= 0"""
     try:
-        if any(int(v) < 0 for v in value.values()):
-            raise ValueError("Must be greater than or equal to zero")
+        for k, v in value.items():
+            if int(v) < 0:
+                raise ValueError("Must be greater than or equal to zero")
+            validate_reserved_names(k)
     except ValueError as err:
         raise ValidationError(str(err))
 
 
-def validate_reserved_names(value):
-    """A value cannot use some reserved names."""
-    if value in settings.RESERVED_NAMES:
-        raise ValidationError('{} is a reserved name.'.format(value))
-
-
 class App(UuidAuditedModel):
     """
     Application used to service requests on behalf of end-users
@@ -112,8 +110,7 @@ class App(UuidAuditedModel):
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     id = models.SlugField(max_length=63, unique=True, null=True,
-                          validators=[validate_app_id,
-                                      validate_reserved_names])
+                          validators=[validate_app_id])
     structure = models.JSONField(
         default=dict, blank=True, validators=[validate_app_structure])
 
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 7a9d29b02..e5f54fcce 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -6,6 +6,7 @@
 from django.db.models import Q
 from django.http import Http404
 
+from api.utils import validate_label
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 
@@ -22,7 +23,7 @@
 class Gateway(AuditedModel):
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
-    name = models.CharField(max_length=63, db_index=True)
+    name = models.CharField(max_length=63, db_index=True, validators=[validate_label])
     ports = models.JSONField(default=list)
 
     def log(self, message, level=logging.INFO):
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 6aebc2b65..b23f19121 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -15,7 +15,7 @@
 
 
 from api import models
-from api.utils import validate_json
+from api.utils import validate_json, validate_reserved_names
 from api.exceptions import DryccException
 from api.models.volume import Volume
 from api.models.base import PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH
@@ -78,6 +78,7 @@ def validate_ptype(value):
         raise serializers.ValidationError(
             "The length of ptype must be between {} and {}".format(
                 PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH))
+    validate_reserved_names(value)
     return value
 
 
@@ -91,6 +92,7 @@ def validate_name(value):
         raise serializers.ValidationError(
             "The length of name must be between {} and {}".format(
                 PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH))
+    validate_reserved_names(value)
     return value
 
 
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 7b5a542e4..5fa65713e 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -255,13 +255,13 @@ def randstr(k):
 TEMPDIR = tempfile.mkdtemp(prefix='drycc')
 
 # names which apps cannot reserve for routing
-RESERVED_NAMES_PATH = os.environ.get(
-    'RESERVED_NAMES_PATH', '/etc/controller/reserved-names.txt')
-if os.path.exists(RESERVED_NAMES_PATH):
-    with open(RESERVED_NAMES_PATH) as f:
-        RESERVED_NAMES = [line.strip() for line in f if line]
+RESERVED_NAME_PATTERNS_PATH = os.environ.get(
+    'RESERVED_NAME_PATTERNS_PATH', '/etc/controller/reserved-name-patterns.txt')
+if os.path.exists(RESERVED_NAME_PATTERNS_PATH):
+    with open(RESERVED_NAME_PATTERNS_PATH) as f:
+        RESERVED_NAME_PATTERNS = [line.strip() for line in f if line.strip()]
 else:
-    RESERVED_NAMES = ["drycc", "drycc-helmbroker", "drycc-manager", "kube-system", "default"]
+    RESERVED_NAME_PATTERNS = [r"^drycc(?:-[\w-]+)?$", r"^kube(?:-[\w-]+)?$", r"^default$"]
 
 # the k8s namespace in which the controller and workflow were installed.
 WORKFLOW_NAMESPACE = os.environ.get('WORKFLOW_NAMESPACE', 'drycc')
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 02895f794..f2ff92811 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -158,8 +158,8 @@ def test_app_errors(self, mock_requests):
 
     def test_app_reserved_names(self, mock_requests):
         """Nobody should be able to create applications with names which are reserved."""
-        reserved_names = ['foo', 'bar']
-        with self.settings(RESERVED_NAMES=reserved_names):
+        reserved_names = ['fooooo', 'barrrrrr']
+        with self.settings(RESERVED_NAME_PATTERNS=reserved_names):
             for name in reserved_names:
                 response = self.client.post('/v2/apps', {'id': name})
                 self.assertContains(
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index a8ea4deef..180f31be0 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -75,6 +75,14 @@ def validate_label(value):
     match = re.match(r'^[a-z0-9-]+$', value)
     if not match:
         raise ValidationError("Can only contain a-z (lowercase), 0-9 and hyphens")
+    validate_reserved_names(value)
+
+
+def validate_reserved_names(value):
+    """A value cannot use some reserved names."""
+    for reserved_name_pattern in settings.RESERVED_NAME_PATTERNS:
+        if re.match(reserved_name_pattern, value):
+            raise ValidationError('{} is a reserved name.'.format(value))
 
 
 def random_string(num):
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 41fb361c9..1aaff30d5 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -14,6 +14,7 @@
 
 from urllib.parse import urljoin
 from asgiref.sync import async_to_sync
+from django.db import transaction
 from django.db.models import Q
 from django.core.cache import cache
 from django.http import Http404, HttpResponse
@@ -348,6 +349,11 @@ def run(self, request, **kwargs):
                 timeout=timeout, expires=expires)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    @transaction.atomic
+    def create(self, request, *args, **kwargs):
+        return super().create(request, *args, **kwargs)
+
+    @transaction.atomic
     def update(self, request, **kwargs):
         app = self.get_object()
         old_owner = app.owner
@@ -361,6 +367,10 @@ def update(self, request, **kwargs):
             downstream_model_owner.delay(app, old_owner, new_owner)
         return Response(status=status.HTTP_200_OK)
 
+    @transaction.atomic
+    def destroy(self, request, *args, **kwargs):
+        return super().destroy(request, *args, **kwargs)
+
 
 class BuildViewSet(ReleasableViewSet):
     """A viewset for interacting with Build objects."""
@@ -1353,7 +1363,7 @@ async def authenticate(self, request, username):
             return None, {'error': 'Unauthorized', "status_code": 401}
         if auth[0].username != username:
             return None, {'error': 'Access denied', "status_code": 403}
-        return auth[0].id, None        
+        return auth[0].id, None
 
 
 @method_decorator(csrf_exempt, name='dispatch')

From 46ca0ad33c7d9fe7081bea0bac792039c8b042eb Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 17 Jul 2025 16:55:30 +0800
Subject: [PATCH 152/200] feat(limitplan): add volume (#189)

---
 ...itplan_container_volume_mounts_and_more.py | 23 +++++++++++++++++++
 rootfs/api/models/app.py                      |  2 ++
 rootfs/api/models/limit.py                    |  2 ++
 3 files changed, 27 insertions(+)
 create mode 100644 rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py

diff --git a/rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py b/rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py
new file mode 100644
index 000000000..d6e8058e2
--- /dev/null
+++ b/rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.22 on 2025-07-17 06:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0023_migration_k8s_alter_volume_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='limitplan',
+            name='container_volume_mounts',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AddField(
+            model_name='limitplan',
+            name='pod_volumes',
+            field=models.JSONField(default=list),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 194cb6df0..643c604ab 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1220,6 +1220,8 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
 
         healthcheck = config.healthcheck.get(ptype, {})
         volumes, volume_mounts = self._get_volumes_and_mounts(ptype, volumes)
+        volumes.extend(limit_plan.pod_volumes)
+        volume_mounts.extend(limit_plan.container_volume_mounts)
         return {
             'tags': config.tags.get(ptype, {}),
             'envs': envs,
diff --git a/rootfs/api/models/limit.py b/rootfs/api/models/limit.py
index 14119d5ac..616c29268 100644
--- a/rootfs/api/models/limit.py
+++ b/rootfs/api/models/limit.py
@@ -117,7 +117,9 @@ class LimitPlan(AuditedModel):
     node_selector = models.JSONField(default=dict)
     runtime_class_name = models.CharField(max_length=63, default="")
     pod_security_context = models.JSONField(default=dict)
+    pod_volumes = models.JSONField(default=list)
     container_security_context = models.JSONField(default=dict)
+    container_volume_mounts = models.JSONField(default=list)
 
     class Meta:
         get_latest_by = 'created'

From 46ef19acce3d8d0349f9584813aafabf3a42f055 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 18 Jul 2025 18:19:36 +0800
Subject: [PATCH 153/200] chore(quickwit): add send app log to quickwit

---
 charts/controller/templates/_helpers.tpl      | 10 ++++
 .../controller-celery-deloyment.yaml          |  2 +-
 rootfs/api/filer.py                           |  7 ++-
 rootfs/api/models/app.py                      | 11 ++--
 rootfs/api/models/appsettings.py              |  2 +
 rootfs/api/models/base.py                     | 24 ++++-----
 rootfs/api/models/gateway.py                  |  3 ++
 rootfs/api/models/release.py                  |  3 +-
 rootfs/api/models/resource.py                 |  2 +
 rootfs/api/models/service.py                  |  2 +
 rootfs/api/models/tls.py                      |  2 +
 rootfs/api/models/volume.py                   |  2 +
 rootfs/api/settings/celery.py                 |  4 ++
 rootfs/api/settings/production.py             |  1 +
 rootfs/api/signals.py                         |  5 +-
 rootfs/api/tasks.py                           | 15 +++++-
 rootfs/api/utils.py                           | 51 +++++++++++++------
 rootfs/scheduler/__init__.py                  |  1 +
 rootfs/scheduler/tests/__init__.py            |  2 +-
 rootfs/scheduler/tests/test_deployments.py    |  2 +-
 rootfs/scheduler/tests/test_hpa.py            |  2 +-
 rootfs/scheduler/tests/test_hpa_12_lower.py   |  2 +-
 rootfs/scheduler/tests/test_job.py            |  2 +-
 rootfs/scheduler/tests/test_pods.py           |  2 +-
 rootfs/scheduler/tests/test_pvc.py            |  2 +-
 rootfs/scheduler/tests/test_rc.py             |  2 +-
 rootfs/scheduler/tests/test_secrets.py        |  2 +-
 rootfs/scheduler/tests/test_services.py       |  2 +-
 rootfs/scheduler/tests/test_svcat.py          |  2 +-
 rootfs/scheduler/tests/test_utils.py          |  2 +-
 rootfs/scheduler/{ => tests}/utils.py         | 32 ------------
 31 files changed, 121 insertions(+), 82 deletions(-)
 rename rootfs/scheduler/{ => tests}/utils.py (80%)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index e579365bc..5f78a658c 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -106,6 +106,14 @@ env:
 - name: WORKFLOW_MANAGER_SECRET_KEY
   value: "{{ .Values.workflowManagerSecretKey }}"
 {{- end }}
+- name: POD_IP
+  valueFrom:
+    fieldRef:
+      fieldPath: status.podIP
+- name: POD_NAME
+  valueFrom:
+    fieldRef:
+      fieldPath: metadata.name
 - name: WORKFLOW_NAMESPACE
   valueFrom:
     fieldRef:
@@ -154,6 +162,8 @@ env:
       name: controller-creds
       key: passport-secret
 {{- end }}
+- name: QUICKWIT_INDEXER_URL
+  value: http://drycc-quickwit-indexer:7280
 - name: QUICKWIT_SEARCHER_URL
   value: http://drycc-quickwit-searcher:7280
 - name: QUICKWIT_LOG_INDEX_PREFIX
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/controller-celery-deloyment.yaml
index 186353088..4ff8ad9e9 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/controller-celery-deloyment.yaml
@@ -47,7 +47,7 @@ spec:
         args:
           - /bin/bash
           - -c
-          - celery --app api worker --queues controller.low,controller.middle,controller.high --autoscale=32,1 --loglevel=WARNING
+          - celery --app api worker --queues controller.low,controller.high,controller.middle --autoscale=32,1 --loglevel=WARNING
         {{- end }}
         {{- with index .Values "celery" "resources" }}
         resources:
diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
index 0e7152ebc..f0d8ebf3c 100644
--- a/rootfs/api/filer.py
+++ b/rootfs/api/filer.py
@@ -5,7 +5,8 @@
 from django.core.cache import cache
 from requests.auth import HTTPBasicAuth
 
-from .utils import random_string, get_session, CacheLock
+from .tasks import send_app_log
+from .utils import random_string, get_httpclient, CacheLock
 
 logger = logging.getLogger(__name__)
 
@@ -21,6 +22,7 @@ def __init__(self, app_id, volume, scheduler):
 
     def log(self, message, level=logging.INFO):
         logger.log(level, "[{}]: {}".format(self.app_id, message))
+        send_app_log.delay(self.app_id, message, level)
 
     @property
     def server(self):
@@ -90,7 +92,8 @@ def request(self, method, server, path="/", **kwargs):
         cache.touch(self.cache_key, timeout=settings.DRYCC_FILER_DURATION)
         url = f"http://{server["address"]}:{self.bind.split(":")[1]}/{path}"
         kwargs["auth"] = HTTPBasicAuth(server["username"], server["password"])
-        return get_session().request(method, url, **kwargs)
+        with get_httpclient() as session:
+            session.request(method, url, **kwargs)
 
     def get(self, path, **kwargs):
         return self.request("GET", self.server, path, **kwargs)
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 643c604ab..b30453976 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -20,7 +20,8 @@
 from django.contrib.auth import get_user_model
 from rest_framework.exceptions import ValidationError
 
-from api.utils import get_session, dict_diff
+from api.tasks import send_app_log
+from api.utils import get_httpclient, dict_diff
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
 from api.utils import (
     CacheLock, DeployLock, generate_app_name, apply_tasks, validate_reserved_names)
@@ -185,6 +186,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.id, message, level)
         logger.log(level, "[{}]: {}".format(self.id, message))
 
     def create(self, *args, **kwargs):  # noqa
@@ -1007,9 +1009,10 @@ def _verify_http_health(self, service, **kwargs):
         response = None
         for _ in range(10):
             try:
-                # http://docs.python-requests.org/en/master/user/advanced/#timeouts
-                response = get_session().get(url, timeout=req_timeout)
-                failed = False
+                with get_httpclient() as session:
+                    # http://docs.python-requests.org/en/master/user/advanced/#timeouts
+                    response = session.get(url, timeout=req_timeout)
+                    failed = False
             except requests.exceptions.RequestException:
                 # In case of a failure where response object is not available
                 failed = True
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index 9a9bf194d..4f9066afe 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -6,6 +6,7 @@
 
 from rest_framework.exceptions import NotFound
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.utils import dict_diff
 from api.exceptions import DryccException, AlreadyExists, UnprocessableEntity
 from .base import UuidAuditedModel
@@ -48,6 +49,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def previous(self):
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 0d4c0bc7c..acedee88d 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -11,7 +11,7 @@
 from django.utils.translation import gettext_lazy as _
 
 
-from api.utils import get_session, validate_json, get_scheduler
+from api.utils import get_httpclient, validate_json, get_scheduler
 
 token_manager_oauth_schema = {
     "$schema": "http://json-schema.org/schema#",
@@ -49,18 +49,18 @@ class Meta:
         abstract = True
 
     def app_log(self, app_id, msg):
-        session = get_session()
-        session.post(
-            "",
-            json={
-                "timestamp": int(time.time()),
-                "log": msg,
-                "kubernetes": {
-                    "namespace": app_id,
-                    "container_name": "drycc-controller",
+        with get_httpclient() as session:
+            session.post(
+                "",
+                json={
+                    "timestamp": int(time.time()),
+                    "log": msg,
+                    "kubernetes": {
+                        "namespace": app_id,
+                        "container_name": "drycc-controller",
+                    }
                 }
-            }
-        )
+            )
 
     @property
     def scheduler(self):
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index e5f54fcce..3750e49b5 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -6,6 +6,7 @@
 from django.db.models import Q
 from django.http import Http404
 
+from api.tasks import send_app_log
 from api.utils import validate_label
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
@@ -35,6 +36,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def add(self, port, protocol):
@@ -255,6 +257,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def refresh_to_k8s(self):
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 68e244437..687574100 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -7,7 +7,7 @@
 from django.db.models import Q
 from django.contrib.auth import get_user_model
 from django.db.models import F, Func, Value, JSONField
-from api.tasks import run_pipeline
+from api.tasks import run_pipeline, send_app_log
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
 from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
@@ -130,6 +130,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def new(self, user, config, build, summary=None):
diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index 10128de37..1de1301c7 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -3,6 +3,7 @@
 from django.db import models, transaction
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException, AlreadyExists, ServiceUnavailable
+from api.tasks import send_app_log
 from api.utils import validate_label, get_scheduler
 from scheduler import KubeException
 from .base import UuidAuditedModel
@@ -124,6 +125,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def bind(self, *args, **kwargs):
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index f5222838b..1651d3110 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -2,6 +2,7 @@
 from functools import partial
 from django.db import models
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.utils import validate_json
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
@@ -131,6 +132,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def refresh_k8s_svc(self):
diff --git a/rootfs/api/models/tls.py b/rootfs/api/models/tls.py
index 8eb98b1c2..9eaa6a081 100644
--- a/rootfs/api/models/tls.py
+++ b/rootfs/api/models/tls.py
@@ -3,6 +3,7 @@
 from django.db import models
 from django.db import transaction
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.exceptions import AlreadyExists
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
@@ -38,6 +39,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     @property
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 58f1ae7fe..26e09ffd8 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -7,6 +7,7 @@
 from django.db import models, transaction
 from django.conf import settings
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.utils import unit_to_bytes, validate_label
 from api.exceptions import DryccException, ServiceUnavailable
 from scheduler import KubeException
@@ -74,6 +75,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def to_measurements(self, timestamp: float):
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
index 9485d238a..b0ef9f858 100644
--- a/rootfs/api/settings/celery.py
+++ b/rootfs/api/settings/celery.py
@@ -58,6 +58,10 @@ class Config(object):
             'queue': 'controller.high',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
+        'api.tasks.send_app_log': {
+            'queue': 'controller.middle',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.middle',
+        },
     },
     task_queues=(
         Queue(
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 5fa65713e..db8c21760 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -477,6 +477,7 @@ def randstr(k):
 }
 
 # Quickwit Configuration
+QUICKWIT_INDEXER_URL = os.environ.get('QUICKWIT_INDEXER_URL', None)
 QUICKWIT_SEARCHER_URL = os.environ.get('QUICKWIT_SEARCHER_URL', None)
 QUICKWIT_LOG_INDEX_PREFIX = os.environ.get('QUICKWIT_LOG_INDEX_PREFIX', None)
 
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index a5fc10f40..07f55a5e8 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -13,7 +13,7 @@
 from django.db.models.signals import post_delete, post_save
 from django.dispatch import receiver
 from django.contrib.auth import get_user_model
-from api.utils import get_session
+from api.utils import get_httpclient
 from api.tasks import send_measurements
 from api.models.app import App
 from api.models.service import Service
@@ -111,7 +111,8 @@ def _hook_release_created(**kwargs):
                 ).hexdigest()
 
             try:
-                get_session().post(url, headers=headers)
+                with get_httpclient() as session:
+                    session.post(url, headers=headers)
                 # just notify with the base URL, disregard the added URL query
                 release.log('Sent deploy hook to {}'.format(deploy_hook))
             except requests.RequestException as e:
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 73bf0154c..7b7f0e73d 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -5,7 +5,7 @@
 from django.core import signals
 from celery import shared_task
 
-from api import manager, models
+from api import utils, manager, models
 from api.exceptions import ServiceUnavailable
 logger = logging.getLogger(__name__)
 
@@ -30,6 +30,19 @@ def send_measurements(measurements: List[Dict[str, str]]):
         signals.request_finished.send(sender=task_id)
 
 
+@shared_task(retry_kwargs={'max_retries': 3})
+def send_app_log(app_id, msg, level=logging.INFO):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        utils.send_app_log(app_id, msg, level)
+    except Exception as e:
+        signals.got_request_exception.send(sender=task_id)
+        raise e
+    else:
+        signals.request_finished.send(sender=task_id)
+
+
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_kwargs={'max_retries': None}
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index 180f31be0..e3c40b9d4 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -18,9 +18,11 @@
 import importlib
 import jsonschema
 from copy import deepcopy
+from urllib.parse import urljoin
 from django.db import models
 from django.conf import settings
 from django.core.cache import cache
+from django.utils import timezone
 from asgiref.sync import sync_to_async
 from requests_toolbelt import user_agent
 from api import __version__ as drycc_version
@@ -30,21 +32,16 @@
 logger = logging.getLogger(__name__)
 
 
-session = None
-
-
-def get_session():
-    global session
-    if session is None:
-        session = requests.Session()
-        session.headers = {
-            # https://toolbelt.readthedocs.org/en/latest/user-agent.html#user-agent-constructor
-            'User-Agent': user_agent('Drycc Controller', drycc_version),
-        }
-        # `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
-        # http://docs.python-requests.org/en/latest/api/#requests.adapters.HTTPAdapter
-        session.mount('http://', requests.adapters.HTTPAdapter(max_retries=10))
-        session.mount('https://', requests.adapters.HTTPAdapter(max_retries=10))
+def get_httpclient():
+    session = requests.Session()
+    session.headers = {
+        # https://toolbelt.readthedocs.org/en/latest/user-agent.html#user-agent-constructor
+        'User-Agent': user_agent('Drycc Controller', drycc_version),
+    }
+    # `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
+    # http://docs.python-requests.org/en/latest/api/#requests.adapters.HTTPAdapter
+    session.mount('http://', requests.adapters.HTTPAdapter(max_retries=10))
+    session.mount('https://', requests.adapters.HTTPAdapter(max_retries=10))
     return session
 
 
@@ -95,6 +92,30 @@ def generate_app_name():
     return "{}-{}".format(random_string(6), random_string(8))
 
 
+def send_app_log(app_id, msg, level=logging.INFO):
+    if not settings.QUICKWIT_INDEXER_URL:
+        return
+    pod_ip = os.environ.get("POD_IP", "unknown")
+    pod_name = os.environ.get("POD_NAME", "unknown")
+    namespace = os.environ.get("WORKFLOW_NAMESPACE", "unknown")
+    docker_id = hashlib.sha256(f"{namespace}:{pod_name}:{pod_ip}".encode("utf-8")).hexdigest()
+    data = {
+        "kubernetes": {
+            "container_name": "drycc-controller",
+            "docker_id": docker_id,
+            "namespace_name": app_id,
+            "pod_name": pod_name,
+        },
+        "log": f"{logging.getLevelName(level)}\t{app_id}\t{msg}",
+        "offset": int(time.time()),
+        "stream": "stdout",
+        "timestamp": timezone.now().isoformat(),
+    }
+    index = f"{settings.QUICKWIT_LOG_INDEX_PREFIX}{app_id}"
+    with get_httpclient() as session:
+        session.post(urljoin(settings.QUICKWIT_INDEXER_URL, f"/api/v1/{index}/ingest"), json=data)
+
+
 def dict_diff(dict1, dict2):
     """
     Returns the added, changed, and deleted items in dict1 compared with dict2.
diff --git a/rootfs/scheduler/__init__.py b/rootfs/scheduler/__init__.py
index 72d3c0376..223364a42 100644
--- a/rootfs/scheduler/__init__.py
+++ b/rootfs/scheduler/__init__.py
@@ -149,6 +149,7 @@ def log(namespace, message, level=logging.INFO):
         sort like releasing or scaling, will be considered as "belonging" to the application
         instead of the controller and will be handled accordingly.
         """
+        utils.send_app_log(namespace, message, level)
         logger.log(level, "[{}]: {}".format(namespace, message))
 
     def http_head(self, path, **kwargs):
diff --git a/rootfs/scheduler/tests/__init__.py b/rootfs/scheduler/tests/__init__.py
index 4e131397d..831c7c0f9 100644
--- a/rootfs/scheduler/tests/__init__.py
+++ b/rootfs/scheduler/tests/__init__.py
@@ -3,7 +3,7 @@
 from django.conf import settings
 
 from scheduler import mock
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class TestCase(DjangoTestCase):
diff --git a/rootfs/scheduler/tests/test_deployments.py b/rootfs/scheduler/tests/test_deployments.py
index c465f9835..fed4705c7 100644
--- a/rootfs/scheduler/tests/test_deployments.py
+++ b/rootfs/scheduler/tests/test_deployments.py
@@ -8,7 +8,7 @@
 from packaging.version import parse, Version, InvalidVersion
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class DeploymentsTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_hpa.py b/rootfs/scheduler/tests/test_hpa.py
index c5a7aa673..831adf47a 100644
--- a/rootfs/scheduler/tests/test_hpa.py
+++ b/rootfs/scheduler/tests/test_hpa.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class HorizontalPodAutoscalersTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_hpa_12_lower.py b/rootfs/scheduler/tests/test_hpa_12_lower.py
index 66ac790ce..1da518ee0 100644
--- a/rootfs/scheduler/tests/test_hpa_12_lower.py
+++ b/rootfs/scheduler/tests/test_hpa_12_lower.py
@@ -8,7 +8,7 @@
 
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 @mock.patch('scheduler.KubeHTTPClient.version', lambda *args: Version('1.2'))
diff --git a/rootfs/scheduler/tests/test_job.py b/rootfs/scheduler/tests/test_job.py
index 93b3cf10f..b1a25f5cb 100644
--- a/rootfs/scheduler/tests/test_job.py
+++ b/rootfs/scheduler/tests/test_job.py
@@ -4,7 +4,7 @@
 Run the tests with './manage.py test scheduler'
 """
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class JobTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_pods.py b/rootfs/scheduler/tests/test_pods.py
index 133f04669..21105997f 100644
--- a/rootfs/scheduler/tests/test_pods.py
+++ b/rootfs/scheduler/tests/test_pods.py
@@ -8,7 +8,7 @@
 from datetime import datetime, timedelta, timezone
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 from scheduler.states import PodState
 
 
diff --git a/rootfs/scheduler/tests/test_pvc.py b/rootfs/scheduler/tests/test_pvc.py
index 64a0ce81d..438b90c15 100644
--- a/rootfs/scheduler/tests/test_pvc.py
+++ b/rootfs/scheduler/tests/test_pvc.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class PVCTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_rc.py b/rootfs/scheduler/tests/test_rc.py
index 02127a418..117556cac 100644
--- a/rootfs/scheduler/tests/test_rc.py
+++ b/rootfs/scheduler/tests/test_rc.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class ReplicationControllersTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_secrets.py b/rootfs/scheduler/tests/test_secrets.py
index 668c0adc1..8e000420a 100644
--- a/rootfs/scheduler/tests/test_secrets.py
+++ b/rootfs/scheduler/tests/test_secrets.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class SecretsTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_services.py b/rootfs/scheduler/tests/test_services.py
index f9a739fbd..dd9ffe101 100644
--- a/rootfs/scheduler/tests/test_services.py
+++ b/rootfs/scheduler/tests/test_services.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class ServicesTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_svcat.py b/rootfs/scheduler/tests/test_svcat.py
index 917fd357a..d61cdcd0f 100644
--- a/rootfs/scheduler/tests/test_svcat.py
+++ b/rootfs/scheduler/tests/test_svcat.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class ServiceCatalogTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_utils.py b/rootfs/scheduler/tests/test_utils.py
index 4b53eb013..cc965e7f9 100644
--- a/rootfs/scheduler/tests/test_utils.py
+++ b/rootfs/scheduler/tests/test_utils.py
@@ -1,5 +1,5 @@
 import unittest
-from scheduler import utils
+from api import utils
 
 
 class TestUtils(unittest.TestCase):
diff --git a/rootfs/scheduler/utils.py b/rootfs/scheduler/tests/utils.py
similarity index 80%
rename from rootfs/scheduler/utils.py
rename to rootfs/scheduler/tests/utils.py
index ed0f273f3..90cf6fdf8 100644
--- a/rootfs/scheduler/utils.py
+++ b/rootfs/scheduler/tests/utils.py
@@ -1,4 +1,3 @@
-from copy import deepcopy
 import random
 
 
@@ -60,34 +59,3 @@ def generate_random_name():
     ]
     return "{}-{}".format(
         random.choice(adjectives), random.choice(nouns))
-
-
-def dict_merge(origin, merge):
-    """
-    Recursively merges dict's. not just simple a["key"] = b["key"], if
-    both a and b have a key who's value is a dict then dict_merge is called
-    on both values and the result stored in the returned dictionary.
-    Also handles merging lists if they occur within the dict
-    """
-    if not isinstance(merge, dict):
-        return merge
-
-    result = deepcopy(origin)
-    for key, value in merge.items():
-        if key in result and isinstance(result[key], dict):
-            result[key] = dict_merge(result[key], value)
-        else:
-            if isinstance(value, list):
-                if key not in result:
-                    result[key] = value
-                else:
-                    # merge lists without leaving potential duplicates
-                    # result[key] = list(set(result[key] + value))  # changes the order as well
-                    for item in value:
-                        if item in result[key]:
-                            continue
-
-                        result[key].append(item)
-            else:
-                result[key] = deepcopy(value)
-    return result

From 35f45da6d90c8cec9696893504e5223e0a546990 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 22 Jul 2025 14:54:41 +0800
Subject: [PATCH 154/200] fix(pipeline): exception msg is None

---
 rootfs/api/models/app.py         | 6 +++---
 rootfs/api/tests/test_config.py  | 5 +++--
 rootfs/api/tests/test_release.py | 3 ++-
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index b30453976..8f027f8f1 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -316,11 +316,11 @@ def pipeline(self, release, ptypes, force_deploy=False):
             ptypes = list(ptypes) if ptypes is not None else ptypes
             release.add_condition(state="succeed", action="pipeline", ptypes=ptypes)
         except Exception as e:
-            release.failed, release.state = True, "crashed"
+            release.failed, release.state, msg = True, "crashed", f"{type(e).__name__}: {e}"
             ptypes = list(ptypes) if ptypes is not None else ptypes
             release.add_condition(
-                state="crashed", action="pipeline", ptypes=ptypes, exception=str(e))
-            self.log(f"{prefix} pipeline runtime error: {release.exception}", logging.ERROR)
+                state="crashed", action="pipeline", ptypes=ptypes, exception=msg)
+            self.log(f"{prefix} pipeline runtime error, {msg}", logging.ERROR)
         finally:
             DeployLock(self.pk).release(ptypes)  # release all locks
             release.save(update_fields=["state", "failed"])  # avoid overwriting other fields
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 6fe1308a8..9cb526531 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -412,7 +412,7 @@ def test_config_deploy_failure(self, mock_requests):
             app = App.objects.get(id=app_id)
             release = app.release_set.latest()
             self.assertEqual(release.failed, True)
-            self.assertEqual(release.conditions[0]['exception'], "Boom!")
+            self.assertEqual(release.conditions[0]['exception'], "Exception: Boom!")
 
     def test_invalid_config_keys(self, mock_requests):
         """Test that invalid config keys are rejected.
@@ -524,7 +524,8 @@ def test_config_failures(self, mock_requests):
             self.assertEqual(response.status_code, 201)
             self.assertEqual(app.release_set.latest().version, 4)
             self.assertEqual(app.release_set.latest().failed, True)
-            self.assertEqual(app.release_set.latest().conditions[0]['exception'], "Boom!")
+            self.assertEqual(
+                app.release_set.latest().conditions[0]['exception'], "Exception: Boom!")
             self.assertEqual(app.release_set.filter(failed=False).latest().version, 3)
 
         # create a build to see that the new release is created with the last successful config
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 4d6f974fd..6330ccdb7 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -132,7 +132,8 @@ def test_conditions(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
             url = f'/v2/apps/{app_id}/releases/'
             response = self.client.get(url)
-            self.assertEqual(response.data['results'][0]['conditions'][0]['exception'], 'Boom!')
+            self.assertEqual(
+                response.data['results'][0]['conditions'][0]['exception'], 'Exception: Boom!')
 
     def test_get_image(self, mock_requests):
         app_id = self.create_app()

From 3ab9575d0c05c5107a46f3907ac9d10a1a5cf948 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 23 Jul 2025 20:32:31 +0800
Subject: [PATCH 155/200] chore(app): add project_id for vm

---
 rootfs/api/serializers/__init__.py |  6 ++--
 rootfs/api/views.py                | 53 +++++-------------------------
 2 files changed, 12 insertions(+), 47 deletions(-)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index b23f19121..3fda7648c 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -148,9 +148,9 @@ class AuthSerializer(serializers.Serializer):
 class UserSerializer(serializers.ModelSerializer):
     class Meta:
         model = User
-        fields = ['email', 'username', 'password', 'first_name', 'last_name', 'is_superuser',
-                  'is_staff', 'groups', 'user_permissions', 'last_login', 'date_joined',
-                  'is_active']
+        fields = ['id', 'email', 'username', 'password', 'first_name', 'last_name',
+                  'is_superuser', 'is_staff', 'groups', 'user_permissions', 'last_login',
+                  'date_joined', 'is_active']
         read_only_fields = ['id', 'is_superuser', 'is_staff', 'groups',
                             'user_permissions', 'last_login', 'date_joined', 'is_active']
         extra_kwargs = {'password': {'write_only': True}}
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 1aaff30d5..f6e1b6000 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -7,13 +7,13 @@
 import json
 import ssl
 import time
+import zlib
 import random
 import aiohttp
 import requests
 import warnings
 
 from urllib.parse import urljoin
-from asgiref.sync import async_to_sync
 from django.db import transaction
 from django.db.models import Q
 from django.core.cache import cache
@@ -1226,42 +1226,6 @@ def _get_app(self):
         self.check_object_permissions(self.request, app)
         return app
 
-    def _get_usage(self, func):
-        def wrap(app_id, ptype, every, start, stop, step):
-            result = {}
-            for item in async_to_sync(func)(app_id, ptype, every, start, stop, step):
-                result[item['metric']['pod']] = item['values']
-            return result
-        return wrap
-
-    @method_decorator(cache_page(settings.DRYCC_METRICS_EXPIRY))
-    @method_decorator(vary_on_headers("Authorization"))
-    def status(self, request, **kwargs):
-        warnings.warn(
-            'this interface will be removed in the next version.', PendingDeprecationWarning)
-        app_id = self._get_app().id
-        data = serializers.MetricSerializer(data=self.request.query_params)
-        if not data.is_valid():
-            return Response(data.errors, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
-        every, start, stop, step = data.validated_data["every"], data.validated_data[
-            'start'], data.validated_data['stop'], data.validated_data['step']
-        params = {
-            "app_id": app_id, "ptype": kwargs['ptype'], "every": every,
-            "start": start, "stop": stop, "step": step,
-        }
-        return Response({
-            "id": app_id,
-            "ptype": kwargs['ptype'],
-            "usage": {
-                "cpus": self._get_usage(monitor.query_cpu_usage)(**params),
-                "memory": self._get_usage(monitor.query_memory_usage)(**params),
-                "networks": {
-                    "receive": self._get_usage(monitor.query_network_receive_usage)(**params),
-                    "transmit": self._get_usage(monitor.query_network_transmit_usage)(**params),
-                }
-            }
-        })
-
     @method_decorator(cache_page(settings.DRYCC_METRICS_EXPIRY))
     @method_decorator(vary_on_headers("Authorization"))
     def metric(self, request, **kwargs):
@@ -1302,16 +1266,17 @@ async def sample(self, name, labels_str, value):
         app_id = labels.get("namespace", None)
         if not app_id:
             return None
-        owner_id, timeout = self.cache.get(app_id, self.default_cache_value)
-        if (owner_id < 0 and timeout < 0) or time.time() > timeout:
+        account_id, timeout = self.cache.get(app_id, self.default_cache_value)
+        if (account_id < 0 and timeout < 0) or time.time() > timeout:
             if app := await models.app.App.objects.filter(id=app_id).afirst():
-                owner_id = app.owner_id
+                account_id = app.owner_id
             else:
-                owner_id = -1
-            self.cache[app_id] = (owner_id, time.time() + random.randint(600, 1200))
-        if owner_id < 0:
+                account_id = -1
+            self.cache[app_id] = (account_id, time.time() + random.randint(600, 1200))
+        if account_id < 0:
             return None
-        labels.update({'vm_project_id': app_id, 'vm_account_id': owner_id})
+        project_id = zlib.crc32(app_id.encode("utf-8"))
+        labels.update({'vm_account_id': account_id, 'vm_project_id': project_id})
         return "%s{%s} %s\n" % (name, ",".join([f'{k}="{v}"' for k, v in labels.items()]), value)
 
     async def get(self, request, node, metrics=None):

From 6cf77d975bcbdcd6a9a85b573397aaccc37c2980 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 24 Jul 2025 15:34:39 +0800
Subject: [PATCH 156/200] feat(controller): add metric runner

---
 charts/controller/templates/_helpers.tpl      | 18 ++++
 .../{ => api}/controller-api-deployment.yaml  | 18 ++--
 .../{ => api}/controller-api-service.yaml     |  5 +-
 .../controller-celery-deloyment.yaml          |  8 +-
 .../templates/controller-cronjob-daily.yaml   |  3 +-
 .../templates/controller-cronjob-hourly.yaml  |  3 +-
 .../templates/controller-mutate-service.yaml  | 11 ---
 .../metric/controller-metric-deployment.yaml  | 87 +++++++++++++++++++
 .../controller-metric-networkpolicy.yaml      | 19 ++++
 .../metric/controller-metric-service.yaml     | 21 +++++
 .../controller-mutate-deloyment.yaml          | 14 +--
 .../mutate/controller-mutate-service.yaml     | 20 +++++
 .../controller-mutate-webhook.yaml            |  0
 charts/controller/values.yaml                 | 32 +++++--
 rootfs/api/settings/production.py             | 18 ++--
 rootfs/api/urls.py                            | 16 ++--
 rootfs/api/views.py                           | 17 ++--
 rootfs/drycc/gunicorn/config.py               | 24 +++--
 18 files changed, 259 insertions(+), 75 deletions(-)
 rename charts/controller/templates/{ => api}/controller-api-deployment.yaml (88%)
 rename charts/controller/templates/{ => api}/controller-api-service.yaml (69%)
 rename charts/controller/templates/{ => celery}/controller-celery-deloyment.yaml (93%)
 delete mode 100644 charts/controller/templates/controller-mutate-service.yaml
 create mode 100644 charts/controller/templates/metric/controller-metric-deployment.yaml
 create mode 100644 charts/controller/templates/metric/controller-metric-networkpolicy.yaml
 create mode 100644 charts/controller/templates/metric/controller-metric-service.yaml
 rename charts/controller/templates/{ => mutate}/controller-mutate-deloyment.yaml (89%)
 create mode 100644 charts/controller/templates/mutate/controller-mutate-service.yaml
 rename charts/controller/templates/{ => mutate}/controller-mutate-webhook.yaml (100%)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 5f78a658c..01f77ca96 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -180,6 +180,24 @@ env:
   value: api.routers.DefaultReplicaRouter
 {{- end }}
 
+{{- define "controller-api.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_CONTROLLER_RUNNER
+  value: api
+{{- end }}
+
+{{- define "controller-metric.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_CONTROLLER_RUNNER
+  value: metric
+{{- end }}
+
+{{- define "controller-mutate.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_CONTROLLER_RUNNER
+  value: mutate
+{{- end }}
+
 {{/* Generate controller config default limit specs */}}
 {{ define "controller.config.defaultLimitSpecs" }}
 - model: api.limitspec
diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/api/controller-api-deployment.yaml
similarity index 88%
rename from charts/controller/templates/controller-api-deployment.yaml
rename to charts/controller/templates/api/controller-api-deployment.yaml
index e92777e5b..c8fdf45f6 100644
--- a/charts/controller/templates/controller-api-deployment.yaml
+++ b/charts/controller/templates/api/controller-api-deployment.yaml
@@ -1,13 +1,13 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: drycc-controller
+  name: drycc-controller-api
   labels:
     heritage: drycc
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
 spec:
-  replicas: {{ .Values.apiReplicas }}
+  replicas: {{ .Values.api.replicas }}
   strategy:
     rollingUpdate:
       maxSurge: 1
@@ -15,11 +15,13 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: drycc-controller-api
+      app: drycc-controller
+      component: drycc-controller-api
   template:
     metadata:
       labels: {{- include "common.labels.standard" . | nindent 8 }}
-        app: drycc-controller-api
+        app: drycc-controller
+        component: drycc-controller-api
     spec:
       affinity:
         podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.api.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.api.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
@@ -27,7 +29,7 @@ spec:
         nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.api.nodeAffinityPreset.type "key" .Values.api.nodeAffinityPreset.key "values" .Values.api.nodeAffinityPreset.values ) | nindent 10 }}
       serviceAccount: drycc-controller
       initContainers:
-      - name: drycc-controller-init
+      - name: drycc-controller-api-init
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
         imagePullPolicy: {{.Values.imagePullPolicy}}
         args:
@@ -35,9 +37,9 @@ spec:
         - -v
         - -u
         - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
-        {{- include "controller.envs" . | indent 8 }}
+        {{- include "controller-api.envs" . | indent 8 }}
       containers:
-      - name: drycc-controller
+      - name: drycc-controller-api
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
         imagePullPolicy: {{.Values.imagePullPolicy}}
         {{- if .Values.diagnosticMode.enabled }}
@@ -74,7 +76,7 @@ spec:
         resources:
           {{- toYaml . | nindent 10 }}
         {{- end }}
-        {{- include "controller.envs" . | indent 8 }}
+        {{- include "controller-api.envs" . | indent 8 }}
         volumeMounts:
         - name: controller-config
           readOnly: false
diff --git a/charts/controller/templates/controller-api-service.yaml b/charts/controller/templates/api/controller-api-service.yaml
similarity index 69%
rename from charts/controller/templates/controller-api-service.yaml
rename to charts/controller/templates/api/controller-api-service.yaml
index 82208897a..189f8d36e 100644
--- a/charts/controller/templates/controller-api-service.yaml
+++ b/charts/controller/templates/api/controller-api-service.yaml
@@ -7,11 +7,14 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
   labels:
+    app: drycc-controller
     heritage: drycc
+    component: drycc-controller-api
 spec:
   ports:
     - name: http
       port: 80
       targetPort: 8000
   selector:
-    app: drycc-controller-api
+    app: drycc-controller
+    component: drycc-controller-api
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/celery/controller-celery-deloyment.yaml
similarity index 93%
rename from charts/controller/templates/controller-celery-deloyment.yaml
rename to charts/controller/templates/celery/controller-celery-deloyment.yaml
index 4ff8ad9e9..73e2b6bbd 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/celery/controller-celery-deloyment.yaml
@@ -7,7 +7,7 @@ metadata:
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
 spec:
-  replicas: {{ .Values.celeryReplicas }}
+  replicas: {{ .Values.celery.replicas }}
   strategy:
     rollingUpdate:
       maxSurge: 1
@@ -15,11 +15,13 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: drycc-controller-celery
+      app: drycc-controller
+      component: drycc-controller-celery
   template:
     metadata:
       labels: {{- include "common.labels.standard" . | nindent 8 }}
-        app: drycc-controller-celery
+        app: drycc-controller
+        component: drycc-controller-celery
     spec:
       affinity:
         podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.celery.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.celery.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index 3e68cc4e8..123544d96 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -17,7 +17,8 @@ spec:
       template:
         metadata:
           labels: {{- include "common.labels.standard" . | nindent 12 }}
-            app: drycc-controller-conjob
+            app: drycc-controller
+            component: drycc-controller-conjob-daily
         spec:
           restartPolicy: OnFailure
           serviceAccount: drycc-controller
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 15987d248..09bf6d177 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -17,7 +17,8 @@ spec:
       template:
         metadata:
           labels: {{- include "common.labels.standard" . | nindent 12 }}
-            app: drycc-controller-conjob
+            app: drycc-controller
+            component: drycc-controller-conjob-hourly
         spec:
           restartPolicy: OnFailure
           serviceAccount: drycc-controller
diff --git a/charts/controller/templates/controller-mutate-service.yaml b/charts/controller/templates/controller-mutate-service.yaml
deleted file mode 100644
index 0621263ef..000000000
--- a/charts/controller/templates/controller-mutate-service.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: drycc-controller-mutate
-spec:
-  selector:
-    component: drycc-controller-mutate
-  ports:
-  - name: https
-    port: 443
-    targetPort: 8443
\ No newline at end of file
diff --git a/charts/controller/templates/metric/controller-metric-deployment.yaml b/charts/controller/templates/metric/controller-metric-deployment.yaml
new file mode 100644
index 000000000..7687889f9
--- /dev/null
+++ b/charts/controller/templates/metric/controller-metric-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: drycc-controller-metric
+  labels:
+    heritage: drycc
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
+spec:
+  replicas: {{ .Values.metric.replicas }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: drycc-controller
+      component: drycc-controller-metric
+  template:
+    metadata:
+      labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app: drycc-controller
+        component: drycc-controller-metric
+    spec:
+      affinity:
+        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.metric.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.metric.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
+        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.metric.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.metric.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
+        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.metric.nodeAffinityPreset.type "key" .Values.metric.nodeAffinityPreset.key "values" .Values.metric.nodeAffinityPreset.values ) | nindent 10 }}
+      serviceAccount: drycc-controller
+      initContainers:
+      - name: drycc-controller-metric-init
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        args:
+        - netcat
+        - -v
+        - -u
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
+        {{- include "controller-metric.envs" . | indent 8 }}
+      containers:
+      - name: drycc-controller-metric
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        {{- if .Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
+        args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if not .Values.diagnosticMode.enabled }}
+        startupProbe:
+          tcpSocket:
+            port: 8000
+          initialDelaySeconds: 30
+          periodSeconds: 60
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 10
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8000
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 8000
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+          periodSeconds: 5
+        {{- end }}
+        ports:
+          - containerPort: 8000
+            name: http
+        {{- with index .Values "metric" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- include "controller-metric.envs" . | indent 8 }}
+        volumeMounts:
+        - name: controller-config
+          readOnly: false
+          mountPath: /etc/controller
+      volumes:
+      - name: controller-config
+        configMap:
+          name: controller-config
\ No newline at end of file
diff --git a/charts/controller/templates/metric/controller-metric-networkpolicy.yaml b/charts/controller/templates/metric/controller-metric-networkpolicy.yaml
new file mode 100644
index 000000000..5ee13a7e3
--- /dev/null
+++ b/charts/controller/templates/metric/controller-metric-networkpolicy.yaml
@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: drycc-controller-metric
+spec:
+  podSelector:
+    matchLabels:
+      app: drycc-controller
+      component: drycc-controller-metric
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: {{.Release.Namespace}}
+      podSelector:
+        matchLabels:
+          app: drycc-victoriametrics
diff --git a/charts/controller/templates/metric/controller-metric-service.yaml b/charts/controller/templates/metric/controller-metric-service.yaml
new file mode 100644
index 000000000..179f08d94
--- /dev/null
+++ b/charts/controller/templates/metric/controller-metric-service.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: drycc-controller-metric
+  annotations:
+    {{- with .Values.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    app: drycc-controller
+    heritage: drycc
+    component: drycc-controller-metric
+spec:
+  clusterIP: None
+  ports:
+    - name: http
+      port: 80
+      targetPort: 8000
+  selector:
+    app: drycc-controller
+    component: drycc-controller-metric
diff --git a/charts/controller/templates/controller-mutate-deloyment.yaml b/charts/controller/templates/mutate/controller-mutate-deloyment.yaml
similarity index 89%
rename from charts/controller/templates/controller-mutate-deloyment.yaml
rename to charts/controller/templates/mutate/controller-mutate-deloyment.yaml
index fbbda5d93..3004bd684 100644
--- a/charts/controller/templates/controller-mutate-deloyment.yaml
+++ b/charts/controller/templates/mutate/controller-mutate-deloyment.yaml
@@ -3,13 +3,15 @@ kind: Deployment
 metadata:
   name: drycc-controller-mutate
 spec:
-  replicas: {{ .Values.mutateReplicas }}
+  replicas: {{ .Values.mutate.replicas }}
   selector:
     matchLabels:
+      app: drycc-controller
       component: drycc-controller-mutate
   template:
     metadata:
       labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app: drycc-controller
         component: drycc-controller-mutate
     spec:
       affinity:
@@ -17,7 +19,7 @@ spec:
         podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.mutate.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.mutate.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
         nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.mutate.nodeAffinityPreset.type "key" .Values.mutate.nodeAffinityPreset.key "values" .Values.mutate.nodeAffinityPreset.values ) | nindent 10 }}
       initContainers:
-      - name: drycc-controller-init
+      - name: drycc-controller-mutate-init
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
         imagePullPolicy: {{.Values.imagePullPolicy}}
         args:
@@ -25,9 +27,9 @@ spec:
         - -v
         - -u
         - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
-        {{- include "controller.envs" . | indent 8 }}
+        {{- include "controller-mutate.envs" . | indent 8 }}
       containers:
-      - name: drycc-controller
+      - name: drycc-controller-mutate
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
         imagePullPolicy: {{.Values.imagePullPolicy}}
         {{- if .Values.diagnosticMode.enabled }}
@@ -56,12 +58,12 @@ spec:
         {{- end }}
         volumeMounts:
         - name: controller-mutate-cert
-          mountPath: /etc/controller/mutate/cert
+          mountPath: /etc/controller/cert
         {{- with index .Values "mutate" "resources" }}
         resources:
           {{- toYaml . | nindent 10 }}
         {{- end }}
-        {{- include "controller.envs" . | indent 8 }}
+        {{- include "controller-mutate.envs" . | indent 8 }}
       volumes:
       - name: controller-mutate-cert                                                                                                                                                                                               
         secret:                                                                                                                                                                                                                           
diff --git a/charts/controller/templates/mutate/controller-mutate-service.yaml b/charts/controller/templates/mutate/controller-mutate-service.yaml
new file mode 100644
index 000000000..120df9a6c
--- /dev/null
+++ b/charts/controller/templates/mutate/controller-mutate-service.yaml
@@ -0,0 +1,20 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: drycc-controller-mutate
+  annotations:
+    {{- with .Values.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    app: drycc-controller
+    heritage: drycc
+    component: drycc-controller-mutate
+spec:
+  selector:
+    app: drycc-controller
+    component: drycc-controller-mutate
+  ports:
+  - name: https
+    port: 443
+    targetPort: 8443
\ No newline at end of file
diff --git a/charts/controller/templates/controller-mutate-webhook.yaml b/charts/controller/templates/mutate/controller-mutate-webhook.yaml
similarity index 100%
rename from charts/controller/templates/controller-mutate-webhook.yaml
rename to charts/controller/templates/mutate/controller-mutate-webhook.yaml
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index b37f0d4d0..1ce7de44c 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -43,12 +43,6 @@ appRuntimeClass: ""
 appDNSPolicy: ""
 # set timeout seconds. It is used for pod exec
 appPodExecTimeout: 3600
-# Set api replicas
-apiReplicas: 1
-# Set celery replicas
-celeryReplicas: 1
-# Set mutate replicas
-mutateReplicas: 1
 # Set cronjob concurrencyPolicy
 # Allow (default): The cron job allows concurrently running jobs
 # Forbid: The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
@@ -92,6 +86,7 @@ service:
 environment: {}
 
 api:
+  replicas: 1
   resources: {}
     # limits:
     #   cpu: 200m
@@ -114,6 +109,7 @@ api:
       app: "drycc-controller-api"
 
 celery:
+  replicas: 1
   resources: {}
     # limits:
     #   cpu: 200m
@@ -135,7 +131,31 @@ celery:
     extraMatchLabels:
       app: "drycc-controller-celery"
 
+metric:
+  replicas: 1
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
+  nodeAffinityPreset:
+    key: "drycc.cc/node"
+    type: "soft"
+    values:
+    - "true"
+  podAffinityPreset:
+    type: ""
+    extraMatchLabels:
+      security: "drycc-security"
+  podAntiAffinityPreset:
+    type: "soft"
+    extraMatchLabels:
+      component: "drycc-controller-metric"
+
 mutate:
+  replicas: 1
   resources: {}
     # limits:
     #   cpu: 200m
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index db8c21760..b686fb253 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -7,7 +7,6 @@
 import random
 import string
 import os.path
-import tempfile
 import dj_database_url
 
 
@@ -250,9 +249,8 @@ def randstr(k):
 }
 TEST_RUNNER = 'api.tests.SilentDjangoTestSuiteRunner'
 
-# default drycc settings
-LOG_LINES = 100
-TEMPDIR = tempfile.mkdtemp(prefix='drycc')
+# drycc controller runner(api, metric, mutate)
+RUNNER = os.environ.get('DRYCC_CONTROLLER_RUNNER', 'api')
 
 # names which apps cannot reserve for routing
 RESERVED_NAME_PATTERNS_PATH = os.environ.get(
@@ -318,13 +316,13 @@ def randstr(k):
 # Drycc service key
 SERVICE_KEY = os.environ.get('DRYCC_SERVICE_KEY', randstr(64))
 
-# Drycc admission mutate key
-MUTATE_KEY_PATH = os.environ.get('DRYCC_MUTATE_KEY_PATH', '/etc/controller/mutate/cert/key')
-if os.path.exists(MUTATE_KEY_PATH):
-    with open(MUTATE_KEY_PATH) as f:
-        MUTATE_KEY = f.read()
+# Drycc cert key
+CERT_KEY_PATH = os.environ.get('DRYCC_CERT_KEY_PATH', '/etc/controller/cert/key')
+if os.path.exists(CERT_KEY_PATH):
+    with open(CERT_KEY_PATH) as f:
+        CERT_KEY = f.read()
 else:
-    MUTATE_KEY = None
+    CERT_KEY = None
 
 IMAGE_PULL_POLICY = os.environ.get('IMAGE_PULL_POLICY', "IfNotPresent")
 
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 92b134fa0..85199e622 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -244,14 +244,9 @@
     re_path(
         r"^apps/(?P<id>{})/perms/(?P<username>[\w.@+-]+)/?$".format(settings.APP_URL_REGEX),
         views.AppPermViewSet.as_view({'put': 'update', 'delete': 'destroy'})),
-    # nodes
-    re_path(
-        r'^nodes/(?P<node>[a-zA-Z0-9-]+)/proxy/metrics(?:/(?P<metrics>[^/]+))?/?$',
-        views.MetricsProxyView.as_view()),
     # quickwit
     re_path(
-        r'^quickwit/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$',
-        views.QuickwitProxyView.as_view()),
+        r'^quickwit/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$', views.QuickwitProxyView.as_view()),
     # prometheus
     re_path(
         r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$',
@@ -263,6 +258,10 @@
         views.TokenViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
 ]
 
+metric_urlpatterns = [
+    re_path(r'^metrics/?$', views.MetricsProxyView.as_view()),
+]
+
 mutate_urlpatterns = [
     re_path(
         r'^mutate/(?P<key>.+)/?$',
@@ -277,8 +276,9 @@
     re_path('', include('social_django.urls', namespace='social')),
 ]
 
-# If there is a mutating admission mutate configuration, use mutate url
-if settings.MUTATE_KEY:
+if settings.RUNNER == 'metric':
+    urlpatterns = metric_urlpatterns
+elif settings.RUNNER == 'mutate':
     urlpatterns = mutate_urlpatterns
 else:
     urlpatterns = app_urlpatterns + social_urlpatterns
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index f6e1b6000..6c392af10 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -220,7 +220,7 @@ class AdmissionWebhookViewSet(GenericViewSet):
     def handle(self, request,  **kwargs):
         key = kwargs['key']
         data = json.loads(request.body.decode("utf8"))["request"]
-        if settings.MUTATE_KEY == key:
+        if settings.CERT_KEY == key:
             allowed = True
             for admission_class in self.admission_classes:
                 admission = admission_class()
@@ -1279,16 +1279,19 @@ async def sample(self, name, labels_str, value):
         labels.update({'vm_account_id': account_id, 'vm_project_id': project_id})
         return "%s{%s} %s\n" % (name, ",".join([f'{k}="{v}"' for k, v in labels.items()]), value)
 
-    async def get(self, request, node, metrics=None):
-        if not metrics:
-            url = f"{settings.SCHEDULER_URL}/api/v1/nodes/{node}/proxy/metrics"
-        else:
-            url = f"{settings.SCHEDULER_URL}/api/v1/nodes/{node}/proxy/metrics/{metrics}"
+    async def get(self, request):
+        params = dict(request.GET)
+        if not set(["host", "port"]).issubset(params.keys()):
+            return HttpResponse(
+                "Error: Required parameter 'host' or 'port' is missing or empty", status=400)
+        host, port = params.pop('host')[0], params.pop('port')[0]
+        scheme, path = params.pop('scheme', ['http'])[0], params.pop('path', ['/metrics'])[0]
+        url = urljoin(f"{scheme}://{host}:{port}", path)
         headers = {"Authorization": request.META.get("HTTP_AUTHORIZATION", "")}
 
         async def stream_response():
             async with aiohttp.ClientSession(connector=self.connector) as session:
-                async with session.get(url, headers=headers) as resp:
+                async with session.get(url, params=params, headers=headers) as resp:
                     async for line_bytes in resp.content:
                         line = line_bytes.decode('utf-8', errors='ignore').strip(' \n')
                         if line.startswith('#') and (match := self.match_meta(line)):
diff --git a/rootfs/drycc/gunicorn/config.py b/rootfs/drycc/gunicorn/config.py
index 7395214e8..9a980075c 100644
--- a/rootfs/drycc/gunicorn/config.py
+++ b/rootfs/drycc/gunicorn/config.py
@@ -4,20 +4,18 @@
 import faulthandler
 faulthandler.enable()
 
-bind = '0.0.0.0:8000'
-
-# If there is a mutating admission mutate configuration, start mutate
-MUTATE_KEY_PATH = os.environ.get(
-    'DRYCC_MUTATE_KEY_PATH', '/etc/controller/mutate/cert/key')
-MUTATE_TLS_KEY_PATH = os.environ.get(
-    'DRYCC_MUTATE_TLS_KEY_PATH', '/etc/controller/mutate/cert/tls.key')
-MUTATE_TLS_CRT_PATH = os.environ.get(
-    'DRYCC_MUTATE_TLS_CRT_PATH', '/etc/controller/mutate/cert/tls.crt')
-if exists(MUTATE_KEY_PATH) and exists(MUTATE_TLS_KEY_PATH) and exists(MUTATE_TLS_CRT_PATH):
+# If a certificate exists, set the certificate and bind to port 8443.
+CERT_KEY_PATH = os.environ.get(
+    'DRYCC_CERT_KEY_PATH', '/etc/controller/cert/key')
+CERT_TLS_KEY_PATH = os.environ.get(
+    'DRYCC_CERT_TLS_KEY_PATH', '/etc/controller/cert/tls.key')
+CERT_TLS_CRT_PATH = os.environ.get(
+    'DRYCC_CERT_TLS_CRT_PATH', '/etc/controller/cert/tls.crt')
+if exists(CERT_KEY_PATH) and exists(CERT_TLS_KEY_PATH) and exists(CERT_TLS_CRT_PATH):
     bind = '0.0.0.0:8443'
-    keyfile = MUTATE_TLS_KEY_PATH
-    certfile = MUTATE_TLS_CRT_PATH
-    reload_extra_files = [MUTATE_KEY_PATH, MUTATE_TLS_KEY_PATH, MUTATE_TLS_CRT_PATH]
+    keyfile = CERT_TLS_KEY_PATH
+    certfile = CERT_TLS_CRT_PATH
+    reload_extra_files = [CERT_KEY_PATH, CERT_TLS_KEY_PATH, CERT_TLS_CRT_PATH]
 else:
     bind = '0.0.0.0:8000'
 

From 7cc04fc6b18fabce7bf6f5a66253e879bfcc390d Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 24 Jul 2025 17:34:30 +0800
Subject: [PATCH 157/200] chore(test): use parallel

---
 charts/controller/templates/_helpers.tpl             |  2 +-
 .../templates/metric/controller-metric-service.yaml  |  2 +-
 rootfs/api/views.py                                  | 12 +++++-------
 rootfs/bin/test-unit                                 |  2 +-
 4 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 01f77ca96..030e92190 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -126,7 +126,7 @@ env:
       key: victoriametrics-url
 {{- else if .Values.victoriametrics.enabled }}
 - name: "DRYCC_VICTORIAMETRICS_URL"
-  value: "http://drycc-victoriametrics-vmselect:8481"
+  value: "http://drycc-victoriametrics-vmselect:8481/select/multitenant/prometheus"
 {{- end }}
 {{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
diff --git a/charts/controller/templates/metric/controller-metric-service.yaml b/charts/controller/templates/metric/controller-metric-service.yaml
index 179f08d94..8151a0cb7 100644
--- a/charts/controller/templates/metric/controller-metric-service.yaml
+++ b/charts/controller/templates/metric/controller-metric-service.yaml
@@ -14,7 +14,7 @@ spec:
   clusterIP: None
   ports:
     - name: http
-      port: 80
+      port: 8000
       targetPort: 8000
   selector:
     app: drycc-controller
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 6c392af10..c6d738ff1 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1325,7 +1325,7 @@ async def authenticate(self, request, username):
         if self.permission.has_permission(request, None):
             if username != "drycc":
                 return None, {'error': 'Access denied', "status_code": 403}
-            return -1, None
+            return 0, None
         auth = await database_sync_to_async(self.authentication.authenticate)(request)
         if not auth or len(auth) != 2:
             return None, {'error': 'Unauthorized', "status_code": 401}
@@ -1448,15 +1448,13 @@ async def proxy(self, request, username, path):
         user_id, message = await self.authenticate(request, username)
         if user_id is None and message is not None:
             return JsonResponse(message, status=message["status_code"])
-        if username == "drycc":
-            path = f"/select/0/prometheus/{path}"
-        else:
-            path = f"/select/{user_id}/prometheus/{path}"
+        data = dict(request.GET) if request.method == "GET" else dict(request.POST)
+        data['extra_filters[]'] = '{vm_account_id="%s"}' % user_id
         try:
             async with aiohttp.ClientSession() as session:
                 async with session.post(
-                    urljoin(settings.DRYCC_VICTORIAMETRICS_URL, path),
-                    data=dict(request.GET) if request.method == "GET" else dict(request.POST),
+                    f"{settings.DRYCC_VICTORIAMETRICS_URL.rstrip("/")}/{path}",
+                    data=data,
                     headers={"Content-Type": "application/x-www-form-urlencoded"},
                     timeout=self.timeout
                 ) as response:
diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit
index 45c343d88..218a94568 100755
--- a/rootfs/bin/test-unit
+++ b/rootfs/bin/test-unit
@@ -15,4 +15,4 @@ function start_valkey() {
 gosu postgres pg_ctl -D "$PGDATA" start && ln -s /tmp/.s.PGSQL.5432 /var/run/postgresql/.s.PGSQL.5432
 start_valkey
 python3 manage.py check
-coverage run --source=api,scheduler manage.py test --settings=api.settings.testing --noinput api scheduler.tests --exclude-tag=auth
+coverage run --source=api,scheduler manage.py test --settings=api.settings.testing --noinput api scheduler.tests --exclude-tag=auth --parallel="${TEST_PARALLEL:-8}"

From 9c7468aaeb5500c0869a025e41df45db4cf90a3a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 28 Jul 2025 16:24:47 +0800
Subject: [PATCH 158/200] chore(gateway): change loadbalancer to gateway
 measure

---
 .../templates/controller-cronjob-hourly.yaml  |  4 +-
 .../management/commands/measure_gateways.py   | 30 +++++++++
 .../commands/measure_loadbalancers.py         | 65 -------------------
 rootfs/api/models/gateway.py                  | 14 ++++
 rootfs/api/monitor.py                         | 11 ----
 5 files changed, 46 insertions(+), 78 deletions(-)
 create mode 100644 rootfs/api/management/commands/measure_gateways.py
 delete mode 100644 rootfs/api/management/commands/measure_loadbalancers.py

diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 09bf6d177..3d2f09b3e 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -59,7 +59,7 @@ spec:
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_loadbalancers
+            - python -u /workspace/manage.py measure_gateway
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
-            {{- include "controller-job.envs" . | indent 12 }}
\ No newline at end of file
+            {{- include "controller-job.envs" . | indent 12 }}
diff --git a/rootfs/api/management/commands/measure_gateways.py b/rootfs/api/management/commands/measure_gateways.py
new file mode 100644
index 000000000..128a08dac
--- /dev/null
+++ b/rootfs/api/management/commands/measure_gateways.py
@@ -0,0 +1,30 @@
+import uuid
+import time
+import logging
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from api.models.gateway import Gateway
+from api.tasks import send_measurements
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Management command for push data to manager"""
+
+    def handle(self, *args, **options):
+        if settings.WORKFLOW_MANAGER_URL:
+            timestamp = time.time()
+            task_id = uuid.uuid4().hex
+            logger.info(f"pushing {task_id} gateways to workflow_manager when {timezone.now()}")
+            gateway_list = []
+            for gateway in Gateway.objects.all():
+                gateway_list.extend(gateway.to_measurements(timestamp))
+                if len(gateway_list) % 1000 == 0:
+                    send_measurements.delay(gateway_list)
+                    gateway_list = []
+            if len(gateway_list) > 0:
+                send_measurements.delay(gateway_list)
+            logger.info(f"pushed {task_id} gateways to workflow_manager when {timezone.now()}")
+            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_loadbalancers.py b/rootfs/api/management/commands/measure_loadbalancers.py
deleted file mode 100644
index 82fb070ca..000000000
--- a/rootfs/api/management/commands/measure_loadbalancers.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import uuid
-import time
-import logging
-import ipaddress
-from asgiref.sync import async_to_sync
-from django.utils import timezone
-from django.core.management.base import BaseCommand
-from django.conf import settings
-from api import monitor
-from api.models.app import App
-from api.tasks import send_measurements
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Management command for push data to manager"""
-
-    def _get_measure_name(self, ip):
-        address = ipaddress.ip_address(ip)
-        prefix = "intranet" if address.is_private else "internet"
-        return f"{prefix}:{address.version}"
-
-    def _measure_loadbalancers(self, app_map, timestamp):
-        stop = timestamp - (timestamp % 3600)
-        start = stop - 3600
-        loadbalancers = []
-        for item in async_to_sync(monitor.query_loadbalancer)(app_map.keys(), start, stop):  # noqa
-            metric = item["metric"]
-            _, value = item["value"]
-            ip = metric["ip"]
-            namespace = metric["namespace"]
-            owner_id = app_map[namespace].owner_id
-            loadbalancers.append({
-                "app_id":  str(app_map[namespace].uuid),
-                "owner": owner_id,
-                "name": self._get_measure_name(ip),
-                "type": "loadbalancer",
-                "unit": "number",
-                "usage": value,
-                "kwargs": {
-                    "ip": ip,
-                    "node": metric["node"],
-                    "service": metric["service"],
-                    "instance": metric["instance"],
-                },
-                "timestamp": start
-            })
-        send_measurements.delay(loadbalancers)
-
-    def handle(self, *args, **options):
-        if settings.WORKFLOW_MANAGER_URL and settings.DRYCC_VICTORIAMETRICS_URL:
-            timestamp = int(time.time())
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
-            app_map = {}
-            for app in App.objects.all():
-                app_map[app.id] = app
-                if len(app_map) % 1000 == 0:
-                    self._measure_loadbalancers(app_map, timestamp)
-                    app_map = {}
-            if len(app_map) > 0:
-                self._measure_loadbalancers(app_map, timestamp)
-            logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
-            self.stdout.write("done")
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 3750e49b5..885b4b31e 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -142,6 +142,20 @@ def delete(self, *args, **kwargs):
             )
         return super().delete(*args, **kwargs)
 
+    def to_measurements(self, timestamp: float):
+        return [{
+            "app_id": str(self.app_id),
+            "owner": self.owner_id,
+            "name": settings.DRYCC_APP_GATEWAY_CLASS,
+            "type": "gateway",
+            "unit": "number",
+            "usage": 1,
+            "kwargs": {
+                "name": self.name,
+            },
+            "timestamp": int(timestamp)
+        }]
+
     def _check_port(self, port, protocol):
         for item in self.ports:
             if item["port"] == port:
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 71d498eb7..d944138d8 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -9,10 +9,6 @@
 last_over_time({__name__=~"%s",namespace="%s"}[%s])
 """
 
-query_loadbalancer_promql_tpl = """
-kube_service_status_load_balancer_ingress{namespace=~"%s"}
-"""
-
 query_network_receive_flow_promql_tpl = """
 increase(container_network_receive_bytes_total{namespace=~"%s"}[%s])
 """
@@ -74,13 +70,6 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
         )
 
 
-async def query_loadbalancer(namespaces: Iterator[str], start: int, stop: int
-                             ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
-    promql = query_loadbalancer_promql_tpl % "|".join(namespaces)
-    return await query_prom(url, {"query": promql, "start": start, "end": stop})
-
-
 async def query_network_receive_flow(namespaces: Iterator[str], start: int, stop: int
                                      ) -> list[tuple[dict[str, str], int]]:
     url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")

From b37e037e719409f6eef8796e7e9185f25fb9ed87 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sat, 2 Aug 2025 20:27:56 +0800
Subject: [PATCH 159/200] chore(manager): rename measurement to usage

---
 .../templates/controller-cronjob-daily.yaml    |  6 +++---
 .../templates/controller-cronjob-hourly.yaml   |  2 +-
 .../{measure_apps.py => upload_app_usage.py}   |  8 ++++----
 ...ure_gateways.py => upload_gateway_usage.py} |  8 ++++----
 ...ure_networks.py => upload_network_usage.py} | 10 +++++-----
 ...e_resources.py => upload_resource_usage.py} |  8 ++++----
 ...asure_volumes.py => upload_volume_usage.py} |  8 ++++----
 rootfs/api/manager.py                          | 16 ++++++++--------
 rootfs/api/models/app.py                       |  8 ++++----
 rootfs/api/models/gateway.py                   |  2 +-
 rootfs/api/models/resource.py                  |  2 +-
 rootfs/api/models/volume.py                    |  2 +-
 rootfs/api/permissions.py                      |  2 +-
 rootfs/api/settings/celery.py                  |  2 +-
 rootfs/api/signals.py                          | 18 +++++++++---------
 rootfs/api/tasks.py                            |  5 ++---
 rootfs/api/tests/test_config.py                |  2 +-
 rootfs/api/tests/test_resource.py              |  4 ++--
 rootfs/api/tests/test_volume.py                |  4 ++--
 19 files changed, 58 insertions(+), 59 deletions(-)
 rename rootfs/api/management/commands/{measure_apps.py => upload_app_usage.py} (80%)
 rename rootfs/api/management/commands/{measure_gateways.py => upload_gateway_usage.py} (80%)
 rename rootfs/api/management/commands/{measure_networks.py => upload_network_usage.py} (89%)
 rename rootfs/api/management/commands/{measure_resources.py => upload_resource_usage.py} (80%)
 rename rootfs/api/management/commands/{measure_volumes.py => upload_volume_usage.py} (80%)

diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index 123544d96..cceeaa48e 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -71,7 +71,7 @@ spec:
             args:
               - /bin/bash
               - -c
-              - python -u /workspace/manage.py measure_apps
+              - python -u /workspace/manage.py upload_app_usage
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
@@ -85,7 +85,7 @@ spec:
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_resources
+            - python -u /workspace/manage.py upload_resource_usage
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
@@ -99,7 +99,7 @@ spec:
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_volumes
+            - python -u /workspace/manage.py upload_volume_usage
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 3d2f09b3e..bb83140b2 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -45,7 +45,7 @@ spec:
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_networks
+            - python -u /workspace/manage.py upload_network_usage
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
diff --git a/rootfs/api/management/commands/measure_apps.py b/rootfs/api/management/commands/upload_app_usage.py
similarity index 80%
rename from rootfs/api/management/commands/measure_apps.py
rename to rootfs/api/management/commands/upload_app_usage.py
index 8b677c361..f9749e308 100644
--- a/rootfs/api/management/commands/measure_apps.py
+++ b/rootfs/api/management/commands/upload_app_usage.py
@@ -5,7 +5,7 @@
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from api.models.app import App
-from api.tasks import send_measurements
+from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
 
@@ -20,11 +20,11 @@ def handle(self, *args, **options):
             logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
             app_list = []
             for app in App.objects.all():
-                app_list.extend(app.to_measurements(timestamp))
+                app_list.extend(app.to_usages(timestamp))
                 if len(app_list) % 1000 == 0:
-                    send_measurements.delay(app_list)
+                    send_usage.delay(app_list)
                     app_list = []
             if len(app_list) > 0:
-                send_measurements.delay(app_list)
+                send_usage.delay(app_list)
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_gateways.py b/rootfs/api/management/commands/upload_gateway_usage.py
similarity index 80%
rename from rootfs/api/management/commands/measure_gateways.py
rename to rootfs/api/management/commands/upload_gateway_usage.py
index 128a08dac..b34d13018 100644
--- a/rootfs/api/management/commands/measure_gateways.py
+++ b/rootfs/api/management/commands/upload_gateway_usage.py
@@ -5,7 +5,7 @@
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from api.models.gateway import Gateway
-from api.tasks import send_measurements
+from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
 
@@ -20,11 +20,11 @@ def handle(self, *args, **options):
             logger.info(f"pushing {task_id} gateways to workflow_manager when {timezone.now()}")
             gateway_list = []
             for gateway in Gateway.objects.all():
-                gateway_list.extend(gateway.to_measurements(timestamp))
+                gateway_list.extend(gateway.to_usages(timestamp))
                 if len(gateway_list) % 1000 == 0:
-                    send_measurements.delay(gateway_list)
+                    send_usage.delay(gateway_list)
                     gateway_list = []
             if len(gateway_list) > 0:
-                send_measurements.delay(gateway_list)
+                send_usage.delay(gateway_list)
             logger.info(f"pushed {task_id} gateways to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_networks.py b/rootfs/api/management/commands/upload_network_usage.py
similarity index 89%
rename from rootfs/api/management/commands/measure_networks.py
rename to rootfs/api/management/commands/upload_network_usage.py
index 0dbd43967..eca1bc0ab 100644
--- a/rootfs/api/management/commands/measure_networks.py
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -7,7 +7,7 @@
 from django.conf import settings
 from api import monitor
 from api.models.app import App
-from api.tasks import send_measurements
+from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
 
@@ -15,7 +15,7 @@
 class Command(BaseCommand):
     """Management command for push data to manager"""
 
-    def _measure_networks(self, app_map, timestamp):
+    def _upload_network_usage(self, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         networks = []
@@ -49,7 +49,7 @@ def _measure_networks(self, app_map, timestamp):
                 },
                 "timestamp": start
             })
-        send_measurements.delay(networks)
+        send_usage.delay(networks)
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
@@ -60,9 +60,9 @@ def handle(self, *args, **options):
             for app in App.objects.all():
                 app_map[app.id] = app
                 if len(app_map) % 1000 == 0:
-                    self._measure_networks(app_map, timestamp)
+                    self._upload_network_usage(app_map, timestamp)
                     app_map = {}
             if len(app_map) > 0:
-                self._measure_networks(app_map, timestamp)
+                self._upload_network_usage(app_map, timestamp)
             logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_resources.py b/rootfs/api/management/commands/upload_resource_usage.py
similarity index 80%
rename from rootfs/api/management/commands/measure_resources.py
rename to rootfs/api/management/commands/upload_resource_usage.py
index b1f9ceccb..78cfc3f43 100644
--- a/rootfs/api/management/commands/measure_resources.py
+++ b/rootfs/api/management/commands/upload_resource_usage.py
@@ -5,7 +5,7 @@
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from api.models.resource import Resource
-from api.tasks import send_measurements
+from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
 
@@ -20,11 +20,11 @@ def handle(self, *args, **options):
             logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
             resource_list = []
             for resource in Resource.objects.filter(status="Ready"):
-                resource_list.extend(resource.to_measurements(timestamp))
+                resource_list.extend(resource.to_usages(timestamp))
                 if len(resource_list) % 1000 == 0:
-                    send_measurements.delay(resource_list)
+                    send_usage.delay(resource_list)
                     resource_list = []
             if len(resource_list) > 0:
-                send_measurements.delay(resource_list)
+                send_usage.delay(resource_list)
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_volumes.py b/rootfs/api/management/commands/upload_volume_usage.py
similarity index 80%
rename from rootfs/api/management/commands/measure_volumes.py
rename to rootfs/api/management/commands/upload_volume_usage.py
index ecf14c524..6bf93ad0c 100644
--- a/rootfs/api/management/commands/measure_volumes.py
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -5,7 +5,7 @@
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from api.models.volume import Volume
-from api.tasks import send_measurements
+from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
 
@@ -20,11 +20,11 @@ def handle(self, *args, **options):
             logger.info(f"pushing {task_id} volumes to workflow_manager when {timezone.now()}")
             volume_list = []
             for volume in Volume.objects.all():
-                volume_list.extend(volume.to_measurements(timestamp))
+                volume_list.extend(volume.to_usages(timestamp))
                 if len(volume_list) % 1000 == 0:
-                    send_measurements.delay(volume_list)
+                    send_usage.delay(volume_list)
                     volume_list = []
             if len(volume_list) > 0:
-                send_measurements.delay(volume_list)
+                send_usage.delay(volume_list)
             logger.info(f"pushed {task_id} volumes to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/manager.py b/rootfs/api/manager.py
index 5b985aabd..a7a178db2 100644
--- a/rootfs/api/manager.py
+++ b/rootfs/api/manager.py
@@ -54,7 +54,7 @@ def delete(self, url, **kwargs):
         return self.request('delete', url, **kwargs)
 
 
-class User(ManagerAPI):
+class UserAPI(ManagerAPI):
 
     def get_status(self, id):
         """
@@ -78,21 +78,21 @@ def get_status(self, id):
         return status
 
 
-class Measurement(ManagerAPI):
+class UsageAPI(ManagerAPI):
 
-    def post(self, measurements: List[Dict[str, str]]):
+    def post(self, usages: List[Dict[str, str]]):
         """
         [
             {
                 "app_id":  "test",
                 "owner": "test",
-                "name": web,
-                "type": "CPU",
-                "unit": "G"
+                "name": "web",
+                "type": "limits",
+                "unit": "std1.large.c1m1",
                 "usage": "2",
                 "timestamp": "1609231998.9103732"
             }
         ]
         """
-        url = "%s/measurements/" % settings.WORKFLOW_MANAGER_URL
-        return super().post(url=url, json=measurements)
+        url = "%s/usages/" % settings.WORKFLOW_MANAGER_URL
+        return super().post(url=url, json=usages)
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 8f027f8f1..3e921b69e 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -724,12 +724,12 @@ def set_application_config(self, release, ptype):
         else:
             self.scheduler.secret.update(self.id, secret_name, secrets_env, labels=labels)
 
-    def to_measurements(self, timestamp: float):
-        measurements = []
+    def to_usages(self, timestamp: float):
+        usage = []
         config = self.config_set.latest()
         for ptype, scale in self.structure.items():
             plan = config.limits.get(ptype)
-            measurements.append({
+            usage.append({
                 "app_id": str(self.uuid),
                 "owner": self.owner_id,
                 "name": plan,
@@ -741,7 +741,7 @@ def to_measurements(self, timestamp: float):
                 },
                 "timestamp": int(timestamp),
             })
-        return measurements
+        return usage
 
     def __str__(self):
         return self.id
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 885b4b31e..5c1dcbfe7 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -142,7 +142,7 @@ def delete(self, *args, **kwargs):
             )
         return super().delete(*args, **kwargs)
 
-    def to_measurements(self, timestamp: float):
+    def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
             "owner": self.owner_id,
diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index 1de1301c7..a25ca397c 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -207,7 +207,7 @@ def retrieve(self, *args, **kwargs):
             self.save()
         return self.status == self.binding == "Ready"
 
-    def to_measurements(self, timestamp: float):
+    def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
             "owner": self.owner_id,
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 26e09ffd8..0068eec81 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -78,7 +78,7 @@ def log(self, message, level=logging.INFO):
         send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
-    def to_measurements(self, timestamp: float):
+    def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
             "owner": self.owner_id,
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index da41a8e05..208df5baf 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -12,7 +12,7 @@ def get_app_status(app):
     if block:
         return False, block.remark
     if settings.WORKFLOW_MANAGER_URL:
-        status = manager.User().get_status(app.owner.pk)
+        status = manager.UserAPI().get_status(app.owner.pk)
         if not status["is_active"]:
             return False, status["message"]
     return True, None
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
index b0ef9f858..e323267c9 100644
--- a/rootfs/api/settings/celery.py
+++ b/rootfs/api/settings/celery.py
@@ -34,7 +34,7 @@ class Config(object):
 app.conf.update(
     timezone=os.environ.get('TZ', 'UTC'),
     task_routes={
-        'api.tasks.send_measurements': {
+        'api.tasks.send_usage': {
             'queue': 'controller.low',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.low',
         },
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index 07f55a5e8..9cd966f09 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -14,7 +14,7 @@
 from django.dispatch import receiver
 from django.contrib.auth import get_user_model
 from api.utils import get_httpclient
-from api.tasks import send_measurements
+from api.tasks import send_usage
 from api.models.app import App
 from api.models.service import Service
 from api.models.gateway import Gateway
@@ -143,8 +143,8 @@ def app_changed_handle(sender, instance: App, created=False, update_fields=None,
     if settings.WORKFLOW_MANAGER_URL and not created and (
             update_fields is not None and "structure" in update_fields):
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.to_usages(timestamp), ],
         )
 
 
@@ -222,8 +222,8 @@ def config_changed_handle(sender, instance: Config, created=False, update_fields
     if settings.WORKFLOW_MANAGER_URL and (
             created or (update_fields is not None and "limits" in update_fields)):
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.app.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.app.to_usages(timestamp), ],
         )
 
 
@@ -232,8 +232,8 @@ def volume_changed_handle(sender, instance: Volume, created=False, update_fields
     # measure volumes to workflow manager
     if settings.WORKFLOW_MANAGER_URL and created:
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.to_usages(timestamp), ],
         )
 
 
@@ -247,6 +247,6 @@ def resource_changed_handle(
                 "plan" in update_fields
             ))):
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.to_usages(timestamp), ],
         )
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 7b7f0e73d..4591dba72 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -17,12 +17,11 @@
     retry_backoff_max=3600,
     retry_kwargs={'max_retries': None}
 )
-def send_measurements(measurements: List[Dict[str, str]]):
+def send_usage(usage: List[Dict[str, str]]):
     task_id = uuid.uuid4().hex
     signals.request_started.send(sender=task_id)
     try:
-        measurement = manager.Measurement()
-        measurement.post(measurements)
+        manager.UsageAPI().post(usage)
     except Exception as e:
         signals.got_request_exception.send(sender=task_id)
         raise e
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 9cb526531..f24e0054a 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -633,7 +633,7 @@ def test_unset_limits_error(self, mock_requests):
     def call_command(self, *args, **kwargs):
         out = StringIO()
         call_command(
-            "measure_apps",
+            "upload_app_usage",
             *args,
             stdout=out,
             stderr=StringIO(),
diff --git a/rootfs/api/tests/test_resource.py b/rootfs/api/tests/test_resource.py
index 1c591d9d0..31a1c4817 100644
--- a/rootfs/api/tests/test_resource.py
+++ b/rootfs/api/tests/test_resource.py
@@ -155,7 +155,7 @@ def call_command(self, *args, **kwargs):
         from django.core.management import call_command
         out = StringIO()
         call_command(
-            "measure_resources",
+            "upload_resource_usage",
             *args,
             stdout=out,
             stderr=StringIO(),
@@ -163,7 +163,7 @@ def call_command(self, *args, **kwargs):
         )
         return out.getvalue()
 
-    def test_measure_resources(self, *args, **kwargs):
+    def test_upload_resource_usage(self, *args, **kwargs):
         # create
         app_id = self.create_app()
         data = [
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index 3cdaaf8e7..bea1e4656 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -277,7 +277,7 @@ def call_command(self, *args, **kwargs):
         from django.core.management import call_command
         out = StringIO()
         call_command(
-            "measure_volumes",
+            "upload_volume_usage",
             *args,
             stdout=out,
             stderr=StringIO(),
@@ -285,7 +285,7 @@ def call_command(self, *args, **kwargs):
         )
         return out.getvalue()
 
-    def test_measure_volumes(self, *args, **kwargs):
+    def test_upload_volume_usage(self, *args, **kwargs):
         # create
         app_id = self.create_app()
         data = [

From 7ec662700767c10ada1d5bfe53d841869d6ab234 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 13 Aug 2025 09:14:37 +0800
Subject: [PATCH 160/200] chore(usage): optimize usage reporting frequency

---
 .../templates/controller-cronjob-daily.yaml   | 42 ----------------
 .../templates/controller-cronjob-hourly.yaml  | 48 +++++++++++++++++--
 .../management/commands/upload_app_usage.py   | 15 ++++--
 .../commands/upload_gateway_usage.py          | 15 ++++--
 .../commands/upload_network_usage.py          | 15 +++---
 .../commands/upload_resource_usage.py         | 15 ++++--
 .../commands/upload_volume_usage.py           | 17 +++++--
 7 files changed, 99 insertions(+), 68 deletions(-)

diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index cceeaa48e..e43456923 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -61,45 +61,3 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
-          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-            imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-apps
-            {{- if .Values.diagnosticMode.enabled }}
-            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
-            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
-            {{- else }}
-            args:
-              - /bin/bash
-              - -c
-              - python -u /workspace/manage.py upload_app_usage
-            {{- end }}
-            {{- include "controller.envs" . | indent 12 }}
-            {{- include "controller-job.envs" . | indent 12 }}
-          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-            imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-resources
-            {{- if .Values.diagnosticMode.enabled }}
-            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
-            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
-            {{- else }}
-            args:
-            - /bin/bash
-            - -c
-            - python -u /workspace/manage.py upload_resource_usage
-            {{- end }}
-            {{- include "controller.envs" . | indent 12 }}
-            {{- include "controller-job.envs" . | indent 12 }}
-          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-            imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-volumes
-            {{- if .Values.diagnosticMode.enabled }}
-            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
-            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
-            {{- else }}
-            args:
-            - /bin/bash
-            - -c
-            - python -u /workspace/manage.py upload_volume_usage
-            {{- end }}
-            {{- include "controller.envs" . | indent 12 }}
-            {{- include "controller-job.envs" . | indent 12 }}
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index bb83140b2..0a5283440 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -37,7 +37,7 @@ spec:
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-networks
+            name: drycc-controller-upload-network-usage
             {{- if .Values.diagnosticMode.enabled }}
             command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
             args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
@@ -51,7 +51,7 @@ spec:
             {{- include "controller-job.envs" . | indent 12 }}
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-loadbalancer
+            name: drycc-controller-upload-gateway-usage
             {{- if .Values.diagnosticMode.enabled }}
             command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
             args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
@@ -59,7 +59,49 @@ spec:
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_gateway
+            - python -u /workspace/manage.py upload_gateway_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-upload-app-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
+            args:
+              - /bin/bash
+              - -c
+              - python -u /workspace/manage.py upload_app_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-upload-resources-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
+            args:
+            - /bin/bash
+            - -c
+            - python -u /workspace/manage.py upload_resource_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-upload-volume-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
+            args:
+            - /bin/bash
+            - -c
+            - python -u /workspace/manage.py upload_volume_usage
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
diff --git a/rootfs/api/management/commands/upload_app_usage.py b/rootfs/api/management/commands/upload_app_usage.py
index f9749e308..f0c3aaa80 100644
--- a/rootfs/api/management/commands/upload_app_usage.py
+++ b/rootfs/api/management/commands/upload_app_usage.py
@@ -1,6 +1,8 @@
 import uuid
 import time
 import logging
+import random
+from datetime import timedelta
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -15,16 +17,21 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
+            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
             logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
             app_list = []
             for app in App.objects.all():
                 app_list.extend(app.to_usages(timestamp))
                 if len(app_list) % 1000 == 0:
-                    send_usage.delay(app_list)
+                    send_usage.apply_async(
+                        args=(app_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    )
                     app_list = []
             if len(app_list) > 0:
-                send_usage.delay(app_list)
+                send_usage.apply_async(
+                    args=(app_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                )
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_gateway_usage.py b/rootfs/api/management/commands/upload_gateway_usage.py
index b34d13018..16c6e657a 100644
--- a/rootfs/api/management/commands/upload_gateway_usage.py
+++ b/rootfs/api/management/commands/upload_gateway_usage.py
@@ -1,6 +1,8 @@
 import uuid
 import time
 import logging
+import random
+from datetime import timedelta
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -15,16 +17,21 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
+            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
             logger.info(f"pushing {task_id} gateways to workflow_manager when {timezone.now()}")
             gateway_list = []
             for gateway in Gateway.objects.all():
                 gateway_list.extend(gateway.to_usages(timestamp))
                 if len(gateway_list) % 1000 == 0:
-                    send_usage.delay(gateway_list)
+                    send_usage.apply_async(
+                        args=(gateway_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    )
                     gateway_list = []
             if len(gateway_list) > 0:
-                send_usage.delay(gateway_list)
+                send_usage.apply_async(
+                    args=(gateway_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                )
             logger.info(f"pushed {task_id} gateways to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_network_usage.py b/rootfs/api/management/commands/upload_network_usage.py
index eca1bc0ab..14c7aaa61 100644
--- a/rootfs/api/management/commands/upload_network_usage.py
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -1,6 +1,8 @@
 import uuid
 import time
 import logging
+import random
+from datetime import timedelta
 from asgiref.sync import async_to_sync
 from django.utils import timezone
 from django.core.management.base import BaseCommand
@@ -15,7 +17,7 @@
 class Command(BaseCommand):
     """Management command for push data to manager"""
 
-    def _upload_network_usage(self, app_map, timestamp):
+    def _upload_network_usage(self, start_time, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         networks = []
@@ -49,20 +51,21 @@ def _upload_network_usage(self, app_map, timestamp):
                 },
                 "timestamp": start
             })
-        send_usage.delay(networks)
+        send_usage.apply_async(
+            args=(networks,), eta=start_time + timedelta(seconds=random.randint(1, 1800))
+        )
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            timestamp = int(time.time())
-            task_id = uuid.uuid4().hex
+            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
             logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
             app_map = {}
             for app in App.objects.all():
                 app_map[app.id] = app
                 if len(app_map) % 1000 == 0:
-                    self._upload_network_usage(app_map, timestamp)
+                    self._upload_network_usage(start_time, app_map, timestamp)
                     app_map = {}
             if len(app_map) > 0:
-                self._upload_network_usage(app_map, timestamp)
+                self._upload_network_usage(start_time, app_map, timestamp)
             logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_resource_usage.py b/rootfs/api/management/commands/upload_resource_usage.py
index 78cfc3f43..b7eb620d3 100644
--- a/rootfs/api/management/commands/upload_resource_usage.py
+++ b/rootfs/api/management/commands/upload_resource_usage.py
@@ -1,6 +1,8 @@
 import uuid
 import time
 import logging
+import random
+from datetime import timedelta
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -15,16 +17,21 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
+            start_time, timestamp, task_id = timezone.now(), time.time(), uuid.uuid4().hex
             logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
             resource_list = []
             for resource in Resource.objects.filter(status="Ready"):
                 resource_list.extend(resource.to_usages(timestamp))
                 if len(resource_list) % 1000 == 0:
-                    send_usage.delay(resource_list)
+                    send_usage.apply_async(
+                        args=(resource_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    )
                     resource_list = []
             if len(resource_list) > 0:
-                send_usage.delay(resource_list)
+                send_usage.apply_async(
+                    args=(resource_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                )
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_volume_usage.py b/rootfs/api/management/commands/upload_volume_usage.py
index 6bf93ad0c..4e776811b 100644
--- a/rootfs/api/management/commands/upload_volume_usage.py
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -1,6 +1,8 @@
 import uuid
 import time
 import logging
+import random
+from datetime import timedelta
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -15,16 +17,21 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} volumes to workflow_manager when {timezone.now()}")
+            start_time, timestamp, task_id = timezone.now(), time.time(), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} volumes to workflow_manager when {start_time}")
             volume_list = []
             for volume in Volume.objects.all():
                 volume_list.extend(volume.to_usages(timestamp))
                 if len(volume_list) % 1000 == 0:
-                    send_usage.delay(volume_list)
+                    send_usage.apply_async(
+                        args=(volume_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    )
                     volume_list = []
             if len(volume_list) > 0:
-                send_usage.delay(volume_list)
+                send_usage.apply_async(
+                    args=(volume_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                )
             logger.info(f"pushed {task_id} volumes to workflow_manager when {timezone.now()}")
             self.stdout.write("done")

From 1948569af5195a17cf2f7d7055b0f49c296269fc Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Mon, 25 Aug 2025 11:35:07 +0800
Subject: [PATCH 161/200] feat(blocklist): blocklist api (#190)

---
 .../templates/controller-clusterrole.yaml     |   6 +
 rootfs/api/backend.py                         |   2 +-
 .../0025_alter_app_id_alter_gateway_name.py   |  25 ++
 .../migrations/0026_app_suspended_state.py    |  18 ++
 rootfs/api/models/app.py                      |   1 +
 rootfs/api/models/blocklist.py                | 231 ++++++++++++++++++
 rootfs/api/tasks.py                           |  16 ++
 rootfs/api/views.py                           |  21 +-
 rootfs/dev_requirements.txt                   |   3 +
 rootfs/scheduler/mock.py                      | 158 +++++++++++-
 rootfs/scheduler/resources/daemonset.py       |  45 ++++
 rootfs/scheduler/resources/statefulset.py     |  45 ++++
 rootfs/scheduler/tests/test_daemonsets.py     |  72 ++++++
 rootfs/scheduler/tests/test_statefulsets.py   |  57 +++++
 14 files changed, 689 insertions(+), 11 deletions(-)
 create mode 100644 rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py
 create mode 100644 rootfs/api/migrations/0026_app_suspended_state.py
 create mode 100644 rootfs/scheduler/resources/daemonset.py
 create mode 100644 rootfs/scheduler/resources/statefulset.py
 create mode 100644 rootfs/scheduler/tests/test_daemonsets.py
 create mode 100644 rootfs/scheduler/tests/test_statefulsets.py

diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index 611129a13..3809127e3 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -51,6 +51,12 @@ rules:
 - apiGroups: ["extensions", "apps"]
   resources: ["deployments/scale", "replicasets/scale"]
   verbs: ["get", "update"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "patch"]
+- apiGroups: ["extensions", "apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "list", "patch"]
 - apiGroups: ["extensions", "autoscaling"]
   resources: ["horizontalpodautoscalers"]
   verbs: ["get", "list", "create", "update", "delete"]
diff --git a/rootfs/api/backend.py b/rootfs/api/backend.py
index f629f15a6..f5108fe3d 100644
--- a/rootfs/api/backend.py
+++ b/rootfs/api/backend.py
@@ -21,7 +21,7 @@ class DryccOIDC(OpenIdConnectAuth):
     USERINFO_URL = settings.SOCIAL_AUTH_DRYCC_USERINFO_URL
     JWKS_URI = settings.SOCIAL_AUTH_DRYCC_JWKS_URI
     OIDC_ENDPOINT = settings.SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT
-    DEFAULT_SCOPE = ['openid']
+    DEFAULT_SCOPE = ['openid', 'profile', 'email']
     EXTRA_DATA = [
         ('id', 'id'),
         ('access_token', 'access_token'),
diff --git a/rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py b/rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py
new file mode 100644
index 000000000..eeb228016
--- /dev/null
+++ b/rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py
@@ -0,0 +1,25 @@
+# Generated by Django 4.2.22 on 2025-08-22 02:48
+
+import api.models.app
+import api.utils
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0024_limitplan_container_volume_mounts_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='app',
+            name='id',
+            field=models.SlugField(max_length=63, null=True, unique=True, validators=[api.models.app.validate_app_id]),
+        ),
+        migrations.AlterField(
+            model_name='gateway',
+            name='name',
+            field=models.CharField(db_index=True, max_length=63, validators=[api.utils.validate_label]),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0026_app_suspended_state.py b/rootfs/api/migrations/0026_app_suspended_state.py
new file mode 100644
index 000000000..a667b809f
--- /dev/null
+++ b/rootfs/api/migrations/0026_app_suspended_state.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.22 on 2025-08-22 02:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0025_alter_app_id_alter_gateway_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='app',
+            name='suspended_state',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 3e921b69e..e286dda62 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -114,6 +114,7 @@ class App(UuidAuditedModel):
                           validators=[validate_app_id])
     structure = models.JSONField(
         default=dict, blank=True, validators=[validate_app_structure])
+    suspended_state = models.JSONField(default=dict, blank=True)
 
     class Meta:
         verbose_name = 'Application'
diff --git a/rootfs/api/models/blocklist.py b/rootfs/api/models/blocklist.py
index 5f5f6eac3..306aa4d09 100644
--- a/rootfs/api/models/blocklist.py
+++ b/rootfs/api/models/blocklist.py
@@ -1,9 +1,16 @@
+import logging
+import functools
 from django.db import models
 from django.contrib.auth import get_user_model
+from scheduler import KubeHTTPException
+from api.exceptions import ServiceUnavailable
+from api.utils import apply_tasks
 from .app import App
 from .base import UuidAuditedModel
 
+
 User = get_user_model()
+logger = logging.getLogger(__name__)
 
 
 class Blocklist(UuidAuditedModel):
@@ -41,3 +48,227 @@ def get_blocklist(cls, app: App):
     class Meta:
         ordering = ['-created']
         unique_together = (("id", "type"),)
+
+    def related_resource_deployments(self, app: App):
+        "get resource deployments"
+        try:
+            deployments = self.scheduler.deployment.get(app.id, labels={}).json()['items']  # noqa
+            if not deployments:
+                deployments = []
+            data = []
+            for d in deployments:
+                item = {
+                    'name': d['metadata']['name'],
+                    'replicas': d['spec'].get("replicas", 0),
+                }
+                data.append(item)
+            data.sort(key=lambda x: x['name'])
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(list resource deployments): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def related_resource_statefulsets(self, app: App):
+        "get resource statefulsets"
+        try:
+            statefulsets = self.scheduler.statefulset.get(app.id, labels={}).json()['items']  # noqa
+            if not statefulsets:
+                statefulsets = []
+            data = []
+            for s in statefulsets:
+                item = {
+                    'name': s['metadata']['name'],
+                    'replicas': s['spec'].get("replicas", 0),
+                }
+                data.append(item)
+            data.sort(key=lambda x: x['name'])
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(list resource statefulsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def related_resource_daemonsets(self, app: App):
+        "get resource daemonsets"
+        try:
+            daemonsets = self.scheduler.daemonset.get(app.id, labels={}).json()['items']  # noqa
+            if not daemonsets:
+                daemonsets = []
+            data = []
+            for d in daemonsets:
+                item = {
+                    'name': d['metadata']['name'],
+                    'affinity': d['spec']['template']['spec'].get('affinity', {}),
+                }
+                data.append(item)
+            data.sort(key=lambda x: x['name'])
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(list resource daemonsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def suspended_state(self, app: App):
+        """
+        Get deployments/statefulsets/daemonsets with labels app.kubernetes.io/managed-by=Helm
+        Store in the suspended_state field of the app model
+        Format:
+        {"deployments": [{"name":"sample1", "replicas": 1}],
+        "statefulsets": [{"name":"sample1", "replicas": 1}],
+        "daemonsets": [{"name":"sample1", "affinity": {}}]}
+        """
+        suspended_state = {}
+        suspended_state['deployments'] = self.related_resource_deployments(app)
+        suspended_state['statefulsets'] = self.related_resource_statefulsets(app)
+        suspended_state['daemonsets'] = self.related_resource_daemonsets(app)
+        return suspended_state
+
+    def scale_resource_deployments(self, app: App, deployment_name: str, replicas=0):
+        """scale deployments"""
+        try:
+            deployment = self.scheduler.deployment.get(app.id, deployment_name).json()
+            self.scheduler.scales.update(app.id, deployment_name, replicas, deployment)
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(scale resource deployments): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def scale_resource_statefulsets(self, app: App, statefulset_name: str, replicas=0):
+        """scale statefulsets"""
+        try:
+            self.scheduler.statefulset.get(app.id, statefulset_name).json()
+            manifest = {
+                'spec': {
+                    'persistentVolumeClaimRetentionPolicy': {
+                        'whenScaled': 'Retain'
+                    },
+                    'replicas': replicas
+                }
+            }
+            self.scheduler.statefulset.patch(app.id, statefulset_name, manifest)
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(scale resource statefulsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def scale_resource_daemonsets(self, app: App, daemonset_name: str, manifest: dict):
+        """set affinity"""
+        try:
+            self.scheduler.daemonset.get(app.id, daemonset_name).json()
+            self.scheduler.daemonset.patch(app.id, daemonset_name, manifest)
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(scale resource daemonsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def scale_resources(self, app: App, suspended_state: dict, scale_type="block"):
+        "scale resources tasks"
+        tasks = []
+        deployments = suspended_state.get('deployments', [])
+        if not deployments:
+            deployments = []
+        for d in deployments:
+            if scale_type == "unblock":
+                replicas = d['replicas']
+            else:
+                replicas = 0
+            tasks.append((
+                functools.partial(
+                    self.scale_resource_deployments,
+                    app=app,
+                    deployment_name=d['name'],
+                    replicas=replicas
+                ),
+                lambda future, name=d["name"]: app.log(
+                    f'{scale_type} scale deployment {name} callback: {future.result()}',
+                )
+            ))
+
+        statefulsets = suspended_state.get('statefulsets', [])
+        if not statefulsets:
+            statefulsets = []
+        for s in statefulsets:
+            if scale_type == "unblock":
+                replicas = s['replicas']
+            else:
+                replicas = 0
+            tasks.append((
+                functools.partial(
+                    self.scale_resource_statefulsets,
+                    app=app,
+                    statefulset_name=s['name'],
+                    replicas=replicas
+                ),
+                lambda future, name=s["name"]: app.log(
+                    f'{scale_type} scale statefulset {name} callback: {future.result()}',
+                )
+            ))
+
+        daemonsets = suspended_state.get('daemonsets', [])
+        if not daemonsets:
+            daemonsets = []
+        for d in daemonsets:
+            if scale_type == "unblock":
+                manifest = {
+                    "spec": {
+                        "template": {
+                            "spec": {
+                                "affinity": d['affinity']
+                            }
+                        }
+                    }
+                }
+            else:
+                manifest = {
+                    "spec": {
+                        "template": {
+                            "spec": {
+                                "affinity": {
+                                    "nodeAffinity": {
+                                        "requiredDuringSchedulingIgnoredDuringExecution": {
+                                            "nodeSelectorTerms": [{
+                                                "matchExpressions": [{
+                                                    "key": "kubernetes.io/hostname",
+                                                    "operator": "In",
+                                                    "values": [
+                                                        "nohostname"
+                                                    ]
+                                                }]
+                                            }]
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            tasks.append((
+                functools.partial(
+                    self.scale_resource_daemonsets,
+                    app=app,
+                    daemonset_name=d['name'],
+                    manifest=manifest
+                ),
+                lambda future, name=d["name"]: app.log(
+                    f'{scale_type} scale daemonset {name} callback: {future.result()}',
+                )
+            ))
+        try:
+            apply_tasks(tasks)
+        except Exception as e:
+            err = f'({scale_type} scale resources): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 4591dba72..9132d6f93 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -158,3 +158,19 @@ def downstream_model_owner(app, old_owner, new_owner):
         raise e
     else:
         signals.request_finished.send(sender=task_id)
+
+
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_kwargs={'max_retries': None}
+)
+def scale_resources(blocklist, app, suspended_state, scale_type):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        blocklist.scale_resources(app, suspended_state, scale_type)
+    except Exception as e:
+        signals.got_request_exception.send(sender=task_id)
+        raise e
+    else:
+        signals.request_finished.send(sender=task_id)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index c6d738ff1..de9be4abf 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -35,7 +35,7 @@
 
 from api import monitor, models, permissions, serializers, viewsets, authentication, __version__
 from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
-    delete_pod
+    delete_pod, scale_resources
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
@@ -183,13 +183,17 @@ class WorkflowManagerViewset(GenericViewSet):
 
     def block(self, request,  **kwargs):
         try:
-            blocklist, _ = models.blocklist.Blocklist.objects.get_or_create(
+            blocklist, created = models.blocklist.Blocklist.objects.get_or_create(
                 id=kwargs['id'],
                 type=models.blocklist.Blocklist.get_type(kwargs["type"]),
                 defaults={"remark": request.data.get("remark")}
             )
             for app in blocklist.related_apps:
-                scale_app.delay(app, app.owner, {key: 0 for key in app.structure.keys()})
+                if created:
+                    app.suspended_state = blocklist.suspended_state(app)
+                    app.save()
+                # scale to 0
+                scale_resources.delay(blocklist, app, app.suspended_state, "block")
             return HttpResponse(status=201)
         except ValueError as e:
             logger.info(e)
@@ -197,10 +201,17 @@ def block(self, request,  **kwargs):
 
     def unblock(self, request,  **kwargs):
         try:
-            models.blocklist.Blocklist.objects.filter(
+            blocklists = models.blocklist.Blocklist.objects.filter(
                 id=kwargs['id'],
                 type=models.blocklist.Blocklist.get_type(kwargs["type"])
-            ).delete()
+            )
+            for blocklist in blocklists:
+                for app in blocklist.related_apps:
+                    # scale to up
+                    scale_resources.delay(blocklist, app, app.suspended_state, "unblock")
+                    app.suspended_state = {}
+                    app.save()
+                blocklist.delete()
             return HttpResponse(status=204)
         except ValueError as e:
             logger.info(e)
diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt
index 9f542ed42..1654cf51a 100644
--- a/rootfs/dev_requirements.txt
+++ b/rootfs/dev_requirements.txt
@@ -8,3 +8,6 @@ flake8==7.2.0
 
 # mock out python-requests, mostly k8s
 requests-mock==1.12.1
+
+# tracebacks
+tblib==3.1.0
\ No newline at end of file
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index 6d1ac5e62..7d016475e 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -18,6 +18,8 @@
 from django.conf import settings
 from django.core.cache import cache
 
+from api import utils
+
 import logging
 logger = logging.getLogger(__name__)
 
@@ -84,6 +86,7 @@ def _acquire(self):
     'persistentvolumeclaims', 'serviceinstances', 'servicebindings',
     'limitranges', 'gateways', 'httproutes', 'tcproutes', 'udproutes',
     'issuers', 'certificates', 'certificaterequests', 'jobs', 'clusterserviceclasses',
+    'statefulsets', 'daemonsets',
 ]
 
 
@@ -902,7 +905,7 @@ def patch(request, context):  # noqa: C901
     data = request.json()
 
     # merge new data into old but keep labels separate in case they changed
-    if 'labels' in data['metadata']:
+    if 'labels' in data.get('metadata', {}):
         labels = data['metadata'].pop('labels')
         item['metadata'].update(data['metadata'])
         data['metadata'] = item['metadata']
@@ -935,6 +938,10 @@ def patch(request, context):  # noqa: C901
     elif resource_type in ['gateways']:
         data['status'] = item['status']
         cache.set(url, data, None)
+    elif resource_type in ['daemonsets', 'statefulsets']:
+        # item data merge-patch
+        data = utils.dict_merge(item, data)
+        cache.set(url, data, None)
     else:
         # Update the individual resource
         change_cache(url, resource_type, data)
@@ -1106,7 +1113,14 @@ def __init__(self, url, k8s_api_verify_tls=True, metadata=None):
             self.ns.get('duplicate')
         except KubeHTTPException:
             self.ns.create('duplicate')
-
+        # node
+        self._setup_node()
+        # daemonset
+        self._setup_daemonset()
+        # statefulset
+        self._setup_statefulset()
+
+    def _setup_node(self):
         try:
             self.node.get('172.17.8.100')
         except KubeHTTPException:
@@ -1183,12 +1197,146 @@ def __init__(self, url, k8s_api_verify_tls=True, metadata=None):
                     }
                 }
             }
-
             name = 'api_v1_nodes_172.17.8.100'
             cache.set(name, data, None)
             cache.set('api_v1_nodes', [name], None)
-        except Exception as e:
-            logger.critical(e)
+
+    def _setup_daemonset(self):
+        data = {
+            "apiVersion": "apps/v1",
+            "kind": "DaemonSet",
+            "metadata": {
+                "name": "drycc",
+                "namespace": "drycc",
+                "labels": {
+                    "app.kubernetes.io/managed-by": "Helm",
+                    "app.kubernetes.io/name": "drycc"
+                }
+            },
+            "spec": {
+                "selector": {
+                    "matchLabels": {
+                        "app.kubernetes.io/name": "drycc"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app.kubernetes.io/managed-by": "Helm",
+                            "app.kubernetes.io/name": "drycc"
+                        }
+                    },
+                    "spec": {
+                        "affinity": {
+                            "nodeAffinity": {
+                                "requiredDuringSchedulingIgnoredDuringExecution": {
+                                    "nodeSelectorTerms": [
+                                        {
+                                            "matchExpressions": [{
+                                                "key": "kubernetes.io/hostname",
+                                                "operator": "In",
+                                                "values": [
+                                                    "server-01",
+                                                    "server-02",
+                                                    "server-03"
+                                                ]}
+                                            ]
+                                        }
+                                    ]
+                                }
+                            }
+                        },
+                        "containers": [{
+                            "image": "registry.drycc.cc/drycc/drycc",
+                            "imagePullPolicy": "IfNotPresent",
+                            "name": "drycc"
+                        }]
+                    }
+                }
+            },
+            "status": {
+                "currentNumberScheduled": 3,
+                "desiredNumberScheduled": 3,
+                "numberAvailable": 3,
+                "numberMisscheduled": 0,
+                "numberReady": 3,
+                "observedGeneration": 2,
+                "updatedNumberScheduled": 3
+            }
+        }
+        name = 'apis_apps_v1_namespaces_drycc_daemonsets_drycc'
+        cache.set(name, data, None)
+        cache.set('apis_apps_v1_namespaces_drycc_daemonsets', [name], None)
+
+    def _setup_statefulset(self):
+        data = {
+            "apiVersion": "apps/v1",
+            "kind": "StatefulSet",
+            "metadata": {
+                "labels": {
+                    "app.kubernetes.io/managed-by": "Helm",
+                    "app.kubernetes.io/name": "drycc"
+                },
+                "name": "drycc",
+                "namespace": "drycc"
+            },
+            "spec": {
+                "persistentVolumeClaimRetentionPolicy": {
+                    "whenDeleted": "Delete",
+                    "whenScaled": "Retain"
+                },
+                "selector": {
+                    "matchLabels": {
+                        "app.kubernetes.io/name": "drycc"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app.kubernetes.io/managed-by": "Helm",
+                            "app.kubernetes.io/name": "drycc"
+                        }
+                    },
+                    "spec": {
+                            "containers": [{
+                                "image": "registry.drycc.cc/drycc/drycc",
+                                "imagePullPolicy": "IfNotPresent",
+                                "name": "drycc"
+                            }]
+                        }
+                    },
+                "volumeClaimTemplates": [{
+                    "apiVersion": "v1",
+                    "kind": "PersistentVolumeClaim",
+                    "metadata": {
+                        "name": "work-data"
+                    },
+                    "spec": {
+                        "accessModes": [
+                            "ReadWriteOnce"
+                        ],
+                        "resources": {
+                            "requests": {
+                                "storage": "1Gi"
+                            }
+                        },
+                        "volumeMode": "Filesystem"
+                    }
+                }]
+            },
+            "status": {
+                "availableReplicas": 2,
+                "collisionCount": 0,
+                "currentReplicas": 2,
+                "observedGeneration": 4,
+                "readyReplicas": 2,
+                "replicas": 2,
+                "updatedReplicas": 2
+            }
+        }
+        name = 'apis_apps_v1_namespaces_drycc_statefulsets_drycc'
+        cache.set(name, data, None)
+        cache.set('apis_apps_v1_namespaces_drycc_statefulsets', [name], None)
 
 
 scheduler.session = session()
diff --git a/rootfs/scheduler/resources/daemonset.py b/rootfs/scheduler/resources/daemonset.py
new file mode 100644
index 000000000..9d7dfbd33
--- /dev/null
+++ b/rootfs/scheduler/resources/daemonset.py
@@ -0,0 +1,45 @@
+import json
+import logging
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class Daemonset(Resource):
+    api_prefix = 'apis'
+    api_version = 'apps/v1'
+
+    def get(self, namespace, name=None, ignore_exception=False, **kwargs):
+        """
+        Fetch a single Daemonsets or a list
+        """
+        url = '/namespaces/{}/daemonsets'
+        args = [namespace]
+        if name is not None:
+            args.append(name)
+            url += '/{}'
+            message = 'get Daemonset "{}" in Namespace "{}"'
+        else:
+            message = 'get Daemonsets in Namespace "{}"'
+
+        url = self.api(url, *args)
+        response = self.http_get(url, params=self.query_params(**kwargs))
+        if self.unhealthy(response.status_code) and not ignore_exception:
+            args.reverse()  # error msg is in reverse order
+            raise KubeHTTPException(response, message, *args)
+
+        return response
+
+    def patch(self, namespace, name, manifest, ignore_exception=False, **kwargs):
+        url = self.api("/namespaces/{}/daemonsets/{}", namespace, name)
+        response = self.http_patch(
+            url,
+            json=manifest,
+            headers={"Content-Type": "application/merge-patch+json"}
+        )
+
+        if self.unhealthy(response.status_code):
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'patch Daemonset "{}"', name)
+        return response
diff --git a/rootfs/scheduler/resources/statefulset.py b/rootfs/scheduler/resources/statefulset.py
new file mode 100644
index 000000000..e7b0cff8d
--- /dev/null
+++ b/rootfs/scheduler/resources/statefulset.py
@@ -0,0 +1,45 @@
+import json
+import logging
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class Statefulset(Resource):
+    api_prefix = 'apis'
+    api_version = 'apps/v1'
+
+    def get(self, namespace, name=None, ignore_exception=False, **kwargs):
+        """
+        Fetch a single Statefulsets or a list
+        """
+        url = '/namespaces/{}/statefulsets'
+        args = [namespace]
+        if name is not None:
+            args.append(name)
+            url += '/{}'
+            message = 'get Statefulset "{}" in Namespace "{}"'
+        else:
+            message = 'get Statefulsets in Namespace "{}"'
+
+        url = self.api(url, *args)
+        response = self.http_get(url, params=self.query_params(**kwargs))
+        if self.unhealthy(response.status_code) and not ignore_exception:
+            args.reverse()  # error msg is in reverse order
+            raise KubeHTTPException(response, message, *args)
+
+        return response
+
+    def patch(self, namespace, name, manifest, ignore_exception=False, **kwargs):
+        url = self.api("/namespaces/{}/statefulsets/{}", namespace, name)
+        response = self.http_patch(
+            url,
+            json=manifest,
+            headers={"Content-Type": "application/merge-patch+json"}
+        )
+
+        if self.unhealthy(response.status_code):
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'patch Statefulset "{}"', name)
+        return response
diff --git a/rootfs/scheduler/tests/test_daemonsets.py b/rootfs/scheduler/tests/test_daemonsets.py
new file mode 100644
index 000000000..f4c8ab985
--- /dev/null
+++ b/rootfs/scheduler/tests/test_daemonsets.py
@@ -0,0 +1,72 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with "./manage.py test scheduler"
+"""
+from scheduler.tests import TestCase
+from scheduler import KubeHTTPException
+
+
+class DaemonsetsTest(TestCase):
+    """Tests scheduler daemonset calls"""
+
+    def test_get_daemonsets(self):
+        response = self.scheduler.daemonset.get('drycc')
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertIn('items', data)
+        self.assertEqual(1, len(data['items']))
+        # simple verify of data
+        self.assertEqual(data['items'][0]['metadata']['name'], 'drycc')
+
+    def test_get_daemonset(self):
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to get Daemonset doesnotexist in Namespace drycc: 404 Not Found'  # noqa
+        ):
+            self.scheduler.daemonset.get('drycc', 'doesnotexist')
+        namespace = 'drycc'
+        name = 'drycc'
+        response = self.scheduler.daemonset.get(namespace, name)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertEqual(data['apiVersion'], 'apps/v1')
+        self.assertEqual(data['kind'], 'DaemonSet')
+        self.assertEqual(data['metadata']['name'], name)
+
+    def test_patch_daemonset(self):
+        namespace = 'drycc'
+        name = 'drycc'
+        manifest_affinity = {
+            "nodeAffinity": {
+                "requiredDuringSchedulingIgnoredDuringExecution": {
+                    "nodeSelectorTerms": [{
+                        "matchExpressions": [{
+                            "key": "kubernetes.io/hostname",
+                            "operator": "In",
+                            "values": [
+                                "nohostname"
+                            ]
+                        }]
+                    }]
+                }
+            }
+        }
+        response = self.scheduler.daemonset.patch(
+            namespace,
+            name,
+            manifest={
+                "spec": {
+                    "template": {
+                        "spec": {
+                            "affinity": manifest_affinity
+                        }
+                    }
+                }
+            },
+        )
+        affinity = response.json()["spec"]["template"]['spec']['affinity']
+        self.assertEqual(
+            affinity,
+            manifest_affinity
+        )
diff --git a/rootfs/scheduler/tests/test_statefulsets.py b/rootfs/scheduler/tests/test_statefulsets.py
new file mode 100644
index 000000000..292b017a0
--- /dev/null
+++ b/rootfs/scheduler/tests/test_statefulsets.py
@@ -0,0 +1,57 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with "./manage.py test scheduler"
+"""
+from scheduler.tests import TestCase
+from scheduler import KubeHTTPException
+
+
+class StatefulsetsTest(TestCase):
+    """Tests scheduler statefulset calls"""
+
+    def test_get_statefulsets(self):
+        response = self.scheduler.statefulset.get('drycc')
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertIn('items', data)
+        self.assertEqual(1, len(data['items']))
+        # simple verify of data
+        self.assertEqual(data['items'][0]['metadata']['name'], 'drycc')
+
+    def test_get_statefulset(self):
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to get Statefulset doesnotexist in Namespace drycc: 404 Not Found'  # noqa
+        ):
+            self.scheduler.statefulset.get('drycc', 'doesnotexist')
+        namespace = 'drycc'
+        name = 'drycc'
+        response = self.scheduler.statefulset.get(namespace, name)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertEqual(data['apiVersion'], 'apps/v1')
+        self.assertEqual(data['kind'], 'StatefulSet')
+        self.assertEqual(data['metadata']['name'], name)
+
+    def test_patch_statefulset(self):
+        namespace = 'drycc'
+        name = 'drycc'
+        manifest = {
+            'spec': {
+                'persistentVolumeClaimRetentionPolicy': {
+                    'whenScaled': 'Retain'
+                },
+                'replicas': 0
+            }
+        }
+        response = self.scheduler.statefulset.patch(
+            namespace,
+            name,
+            manifest=manifest,
+        )
+        when_scaled = response.json()["spec"]["persistentVolumeClaimRetentionPolicy"]['whenScaled']
+        self.assertEqual(
+            when_scaled,
+            'Retain'
+        )

From 393f43dacb72d89cfc30e64070bada3da50f0e00 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 27 Aug 2025 09:19:07 +0800
Subject: [PATCH 162/200] chore(volumes): usage-based billing

---
 charts/controller/templates/_helpers.tpl      |  9 ++++
 .../commands/upload_network_usage.py          |  4 +-
 .../commands/upload_volume_usage.py           | 53 +++++++++++++------
 rootfs/api/monitor.py                         | 20 +++++++
 rootfs/scheduler/resources/pv.py              |  3 +-
 rootfs/scheduler/resources/pvc.py             |  3 +-
 6 files changed, 71 insertions(+), 21 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 030e92190..dc6547931 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -416,6 +416,9 @@ oss:
 {{/* Generate controller config default volume claim template */}}
 {{ define "controller.config.defaultVolumeClaimTemplate" }}
 csi:
+  metadata:
+    annotations:
+      billing.drycc.cc/type: usage
   spec:
     accessModes:
     - ReadWriteMany
@@ -425,6 +428,9 @@ csi:
         storage: $size
     volumeMode: Filesystem
 nfs:
+  metadata:
+    annotations:
+      billing.drycc.cc/type: basic
   spec:
     accessModes:
     - ReadWriteMany
@@ -434,6 +440,9 @@ nfs:
         storage: $size
     volumeName: $volume_name
 oss:
+  metadata:
+    annotations:
+      billing.drycc.cc/type: basic
   spec:
     accessModes:
     - ReadWriteMany
diff --git a/rootfs/api/management/commands/upload_network_usage.py b/rootfs/api/management/commands/upload_network_usage.py
index 14c7aaa61..b748fe0da 100644
--- a/rootfs/api/management/commands/upload_network_usage.py
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -58,7 +58,7 @@ def _upload_network_usage(self, start_time, app_map, timestamp):
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
             start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
+            logger.info(f"pushing {task_id} networks to workflow_manager when {timezone.now()}")
             app_map = {}
             for app in App.objects.all():
                 app_map[app.id] = app
@@ -67,5 +67,5 @@ def handle(self, *args, **options):
                     app_map = {}
             if len(app_map) > 0:
                 self._upload_network_usage(start_time, app_map, timestamp)
-            logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
+            logger.info(f"pushed {task_id} networks to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_volume_usage.py b/rootfs/api/management/commands/upload_volume_usage.py
index 4e776811b..072ca7831 100644
--- a/rootfs/api/management/commands/upload_volume_usage.py
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -3,10 +3,12 @@
 import logging
 import random
 from datetime import timedelta
+from asgiref.sync import async_to_sync
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
-from api.models.volume import Volume
+from api import monitor
+from api.models.app import App
 from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
@@ -15,23 +17,40 @@
 class Command(BaseCommand):
     """Management command for push data to manager"""
 
+    def _upload_volume_usage(self, start_time, app_map, timestamp):
+        stop = timestamp - (timestamp % 3600)
+        start = stop - 3600
+        volumes = []
+        for item in async_to_sync(monitor.query_volume_size)(app_map.keys(), start, stop):  # noqa
+            metric = item["metric"]
+            _, value = item["value"]
+            volumes.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "owner": app_map[metric['namespace']].owner_id,
+                "type": "volume",
+                "unit": "bytes",
+                "name": metric["storageclass"],
+                "usage": value,
+                "kwargs": {
+                    "persistentvolumeclaim": metric['persistentvolumeclaim'],
+                },
+                "timestamp": start
+            })
+        send_usage.apply_async(
+            args=(volumes,), eta=start_time + timedelta(seconds=random.randint(1, 1800))
+        )
+
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            start_time, timestamp, task_id = timezone.now(), time.time(), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} volumes to workflow_manager when {start_time}")
-            volume_list = []
-            for volume in Volume.objects.all():
-                volume_list.extend(volume.to_usages(timestamp))
-                if len(volume_list) % 1000 == 0:
-                    send_usage.apply_async(
-                        args=(volume_list,),
-                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
-                    )
-                    volume_list = []
-            if len(volume_list) > 0:
-                send_usage.apply_async(
-                    args=(volume_list,),
-                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
-                )
+            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} volumes to workflow_manager when {timezone.now()}")
+            app_map = {}
+            for app in App.objects.all():
+                app_map[app.id] = app
+                if len(app_map) % 1000 == 0:
+                    self._upload_volume_usage(start_time, app_map, timestamp)
+                    app_map = {}
+            if len(app_map) > 0:
+                self._upload_volume_usage(start_time, app_map, timestamp)
             logger.info(f"pushed {task_id} volumes to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index d944138d8..baa47ae73 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -28,6 +28,18 @@
 by (pod)
 """
 
+query_volume_size_promql_tpl = """
+max by(namespace, persistentvolumeclaim, storageclass) (
+  kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="%s"}
+  * on(namespace, persistentvolumeclaim) group_left(storageclass)
+  kube_persistentvolumeclaim_info{namespace="%s"}
+  * on(namespace, persistentvolumeclaim) group_left()
+  kube_persistentvolumeclaim_annotations{
+    namespace="%s",
+    annotation_billing_drycc_cc_type="usage"
+  }
+)
+"""
 
 query_network_receive_usage_promql_tpl = """
 sum (rate (container_network_receive_bytes_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
@@ -70,6 +82,14 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
         )
 
 
+async def query_volume_size(namespaces: Iterator[str], start: int, stop: int
+                            ) -> list[tuple[dict[str, str], int]]:
+    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
+    promql = query_volume_size_promql_tpl % (
+        "|".join(namespaces), "|".join(namespaces), "|".join(namespaces))
+    return await query_prom(url, {"query": promql, "start": start, "end": stop})
+
+
 async def query_network_receive_flow(namespaces: Iterator[str], start: int, stop: int
                                      ) -> list[tuple[dict[str, str], int]]:
     url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
diff --git a/rootfs/scheduler/resources/pv.py b/rootfs/scheduler/resources/pv.py
index 947d1c490..56f6a9e27 100644
--- a/rootfs/scheduler/resources/pv.py
+++ b/rootfs/scheduler/resources/pv.py
@@ -1,5 +1,6 @@
 import json
 import logging
+from api import utils
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -19,7 +20,7 @@ def manifest(self, name, version=None, **kwargs):
                 'labels': labels
             },
         }
-        data.update(kwargs)
+        data = utils.dict_merge(data, kwargs)
         if version:
             data["metadata"]["resourceVersion"] = version
         return data
diff --git a/rootfs/scheduler/resources/pvc.py b/rootfs/scheduler/resources/pvc.py
index df5aa7b40..8582f2016 100644
--- a/rootfs/scheduler/resources/pvc.py
+++ b/rootfs/scheduler/resources/pvc.py
@@ -1,5 +1,6 @@
 import json
 import logging
+from api import utils
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -20,7 +21,7 @@ def manifest(self, namespace, name, version=None, **kwargs):
                 'labels': labels
             }
         }
-        data.update(kwargs)
+        data = utils.dict_merge(data, kwargs)
         if version:
             data["metadata"]["resourceVersion"] = version
         return data

From 073ca6d74bd2bf696a5465df1d9b901916d3d826 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 29 Aug 2025 12:43:05 +0800
Subject: [PATCH 163/200] chore(charts): celery process isolation

---
 .../templates/celery/controller-celery-deloyment.yaml  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/charts/controller/templates/celery/controller-celery-deloyment.yaml b/charts/controller/templates/celery/controller-celery-deloyment.yaml
index 73e2b6bbd..5bf69e54b 100644
--- a/charts/controller/templates/celery/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/celery/controller-celery-deloyment.yaml
@@ -39,7 +39,8 @@ spec:
           - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
         {{- include "controller.envs" . | indent 8 }}
       containers:
-      - name: drycc-controller-celery
+      {{- range $key := (list "low" "middle" "high") }}
+      - name: drycc-controller-celery-{{$key}}
         image: {{$.Values.imageRegistry}}/{{$.Values.imageOrg}}/controller:{{$.Values.imageTag}}
         imagePullPolicy: {{$.Values.imagePullPolicy}}
         {{- if $.Values.diagnosticMode.enabled }}
@@ -49,9 +50,9 @@ spec:
         args:
           - /bin/bash
           - -c
-          - celery --app api worker --queues controller.low,controller.high,controller.middle --autoscale=32,1 --loglevel=WARNING
+          - celery --app api worker --queues controller.{{$key}} --autoscale=32,1 --loglevel=WARNING
         {{- end }}
-        {{- with index .Values "celery" "resources" }}
+        {{- with index $.Values "celery" "resources" }}
         resources:
           {{- toYaml . | nindent 10 }}
         {{- end }}
@@ -60,7 +61,8 @@ spec:
         - name: controller-config
           readOnly: false
           mountPath: /etc/controller
+      {{- end }}
       volumes:
       - name: controller-config
         configMap:
-          name: controller-config
\ No newline at end of file
+          name: controller-config

From 77b33e7ed70f5462c7dd5195881a9d1e5c216861 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Mon, 1 Sep 2025 17:14:41 +0800
Subject: [PATCH 164/200] chore(controller): App add field confict (#191)

---
 .../migrations/0013_migration_permissions_and_certificates.py  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
index f5d4644e0..f8704081b 100644
--- a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -2,12 +2,13 @@
 
 from django.db import migrations
 from guardian.shortcuts import assign_perm, get_users_with_perms, remove_perm
-from api.models.app import App, VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
+from api.models.app import VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
 from api.models.domain import Domain
 from api.models.certificate import Certificate
 
 
 def migration_permission(apps, schema_editor):
+    App = apps.get_model('api', 'App')
     for app in App.objects.all():
         for user in get_users_with_perms(app):
             remove_perm('use_app', user, app)

From f40a6c5bc5f1ad3f942147bb06f550884597b387 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 2 Sep 2025 10:07:50 +0800
Subject: [PATCH 165/200] chore(controller): add .venv to .gitignore

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 1eb305349..2d9ec853d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ var
 sdist
 develop-eggs
 .installed.cfg
+.venv
 lib
 lib64
 
@@ -44,4 +45,4 @@ vendor/
 # generated bintray scripts during ci
 client/_scripts/ci/bintray-ci.json
 .idea
-.vscode/
\ No newline at end of file
+.vscode/

From 3a3a952d07d848dee587b5a1c36d82e9c9562370 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 2 Sep 2025 10:09:47 +0800
Subject: [PATCH 166/200] chore(charts): add celery random worker name

---
 .../templates/celery/controller-celery-deloyment.yaml           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/charts/controller/templates/celery/controller-celery-deloyment.yaml b/charts/controller/templates/celery/controller-celery-deloyment.yaml
index 5bf69e54b..ea9211efe 100644
--- a/charts/controller/templates/celery/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/celery/controller-celery-deloyment.yaml
@@ -50,7 +50,7 @@ spec:
         args:
           - /bin/bash
           - -c
-          - celery --app api worker --queues controller.{{$key}} --autoscale=32,1 --loglevel=WARNING
+          - celery --app api worker -n {{uuidv4}}@%h --queues controller.{{$key}} --autoscale=32,1 --loglevel=WARNING
         {{- end }}
         {{- with index $.Values "celery" "resources" }}
         resources:

From a0d86e28450eff05dab226aa06a4a4b5b7adbbba Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 3 Sep 2025 15:39:26 +0800
Subject: [PATCH 167/200] chore(controller): unify victoriametricsUrl
 configuration format (#192)

---
 charts/controller/values.yaml |  1 +
 rootfs/api/monitor.py         | 17 ++++++++---------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 1ce7de44c..df5ab587f 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -58,6 +58,7 @@ passportUrl: ""
 passportKey: ""
 passportSecret: ""
 # victoriametricsUrl is will no longer use the built-in victoriametrics component
+# format: http://victoriametrics-server/select/0/prometheus
 victoriametricsUrl: ""
 # Workflow-manager Configuration Options
 workflowManagerUrl: ""
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index baa47ae73..9752aed1d 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -1,7 +1,6 @@
 import time
 import aiohttp
 from typing import Iterator, AsyncGenerator
-from urllib.parse import urljoin
 from django.conf import settings
 
 
@@ -66,7 +65,7 @@ async def query_prom(url, params) -> list[tuple[dict[str, str], int]]:
 async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
     if not settings.DRYCC_METRICS_CONFIG:
         return
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
     promql = query_last_metrics_promql_tpl % (
       '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
       namespace,
@@ -84,7 +83,7 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
 
 async def query_volume_size(namespaces: Iterator[str], start: int, stop: int
                             ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
     promql = query_volume_size_promql_tpl % (
         "|".join(namespaces), "|".join(namespaces), "|".join(namespaces))
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
@@ -92,14 +91,14 @@ async def query_volume_size(namespaces: Iterator[str], start: int, stop: int
 
 async def query_network_receive_flow(namespaces: Iterator[str], start: int, stop: int
                                      ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
     promql = query_network_receive_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
 
 async def query_network_transmit_flow(namespaces: Iterator[str], start: int, stop: int
                                       ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
     promql = query_network_transmit_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
@@ -107,7 +106,7 @@ async def query_network_transmit_flow(namespaces: Iterator[str], start: int, sto
 async def query_cpu_usage(namespace: str, ptype: str, every: str,
                           start: int, stop: int, step: int,
                           ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_cpu_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
@@ -116,7 +115,7 @@ async def query_cpu_usage(namespace: str, ptype: str, every: str,
 async def query_memory_usage(namespace: str, ptype: str, every: str,
                              start: int, stop: int, step: int,
                              ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_memory_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
@@ -125,7 +124,7 @@ async def query_memory_usage(namespace: str, ptype: str, every: str,
 async def query_network_receive_usage(namespace: str, ptype: str, every: str,
                                       start: int, stop: int, step: int,
                                       ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_network_receive_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
@@ -134,7 +133,7 @@ async def query_network_receive_usage(namespace: str, ptype: str, every: str,
 async def query_network_transmit_usage(namespace: str, ptype: str, every: str,
                                        start: int, stop: int, step: int,
                                        ) -> list[tuple[dict[str, str], int]]:
-    url = urljoin(settings.DRYCC_VICTORIAMETRICS_URL, "/select/0/prometheus/api/v1/query_range")
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
     pod_prefix = "%s-%s" % (namespace, ptype)
     promql = query_network_transmit_usage_promql_tpl % (pod_prefix, namespace, every)
     return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})

From dec6d93d4da95756b111a0908abd6c6a068ea5f1 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 4 Sep 2025 09:19:33 +0800
Subject: [PATCH 168/200] chore(charts): use randAlphaNum replace randAscii

---
 charts/controller/templates/controller-secret-creds.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index 41723ce98..0e78d51c6 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -31,6 +31,6 @@ data:
   registry-username: {{ .Values.registryUsername | b64enc }}
   registry-password: {{ .Values.registryPassword | b64enc }}
   {{- end }}
-  service-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "service-key" "defaultValue" (randAscii 64) "context" $)) }}
-  django-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "django-secret-key" "defaultValue" (randAscii 64) "context" $)) }}
-  deploy-hook-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "deploy-hook-secret-key" "defaultValue" (randAscii 64) "context" $)) }}
+  service-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "service-key" "defaultValue" (randAlphaNum 64) "context" $)) }}
+  django-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "django-secret-key" "defaultValue" (randAlphaNum 64) "context" $)) }}
+  deploy-hook-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "deploy-hook-secret-key" "defaultValue" (randAlphaNum 64) "context" $)) }}

From 106b32186184b9c71cc96b6c8aa2328d3d699e4f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 5 Sep 2025 13:36:51 +0800
Subject: [PATCH 169/200] chore(usage): add usage promql template

---
 charts/controller/templates/_helpers.tpl      |  29 +++++
 .../templates/controller-configmap.yaml       |  12 ++
 charts/controller/values.yaml                 |   2 +
 .../commands/upload_network_usage.py          |  19 +--
 .../commands/upload_volume_usage.py           |   2 +-
 rootfs/api/monitor.py                         | 121 ++++--------------
 rootfs/api/settings/production.py             |  16 +++
 7 files changed, 84 insertions(+), 117 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index dc6547931..940231ef5 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -452,3 +452,32 @@ oss:
         storage: $size
     volumeName: $volume_name
 {{- end }}
+
+{{/* Generate controller config default volume usage template */}}
+{{ define "controller.config.defaultVolumeUsageTemplate" }}
+max by(namespace, persistentvolumeclaim, storageclass) (
+  kube_persistentvolumeclaim_resource_requests_storage_bytes{
+    namespace=~"${namespaces}"
+  }
+  * on(namespace, persistentvolumeclaim) group_left(storageclass)
+  kube_persistentvolumeclaim_info{namespace=~"${namespaces}"}
+  * on(namespace, persistentvolumeclaim) group_left()
+  kube_persistentvolumeclaim_annotations{
+    namespace=~"${namespaces}",
+    annotation_billing_drycc_cc_type="usage"
+  }
+)
+{{- end }}
+
+{{/* Generate controller config default network usage template */}}
+{{ define "controller.config.defaultNetworkUsageTemplate" }}
+label_replace(
+  sum(increase(istio_request_bytes_sum{namespace=~"${namespaces}"}[${duration}])) by (namespace,pod),
+  "direction", "rx", "", ""
+)
+or
+label_replace(
+  sum(increase(istio_response_bytes_sum{namespace=~"${namespaces}"}[${duration}])) by (namespace,pod),
+  "direction", "tx", "", ""
+)
+{{- end }}
diff --git a/charts/controller/templates/controller-configmap.yaml b/charts/controller/templates/controller-configmap.yaml
index 2a4cdca92..d164f2105 100644
--- a/charts/controller/templates/controller-configmap.yaml
+++ b/charts/controller/templates/controller-configmap.yaml
@@ -47,3 +47,15 @@ data:
     {{- else}}
     {{- include "controller.config.defaultVolumeClaimTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
     {{- end }}
+  volume-usage-template.promql: |
+    {{- if .Values.config.volumeUsageTemplate }}
+    {{- (tpl .Values.config.volumeUsageTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultVolumeUsageTemplate" . | nindent 4 }}
+    {{- end }}
+  network-usage-template.promql: |
+    {{- if .Values.config.networkUsageTemplate }}
+    {{- (tpl .Values.config.networkUsageTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- include "controller.config.defaultNetworkUsageTemplate" . | nindent 4 }}
+    {{- end }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index df5ab587f..2a93156ff 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -75,6 +75,8 @@ config:
   secretTemplate: ""
   volumeTemplate: ""
   volumeClaimTemplate: ""
+  volumeUsageTemplate: ""
+  networkUsageTemplate: ""
 
 # Service
 service:
diff --git a/rootfs/api/management/commands/upload_network_usage.py b/rootfs/api/management/commands/upload_network_usage.py
index b748fe0da..e82fbe7e0 100644
--- a/rootfs/api/management/commands/upload_network_usage.py
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -21,7 +21,7 @@ def _upload_network_usage(self, start_time, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         networks = []
-        for item in async_to_sync(monitor.query_network_receive_flow)(app_map.keys(), start, stop):  # noqa
+        for item in async_to_sync(monitor.query_network_usage)(app_map.keys(), start, stop):
             metric = item["metric"]
             _, value = item["value"]
             networks.append({
@@ -29,22 +29,7 @@ def _upload_network_usage(self, start_time, app_map, timestamp):
                 "owner": app_map[metric['namespace']].owner_id,
                 "type": "network",
                 "unit": "bytes",
-                "name": "rx",
-                "usage": value,
-                "kwargs": {
-                    "pod": metric['pod'],
-                },
-                "timestamp": start
-            })
-        for item in async_to_sync(monitor.query_network_transmit_flow)(app_map.keys(), start, stop):  # noqa
-            metric = item["metric"]
-            _, value = item["value"]
-            networks.append({
-                "app_id":  str(app_map[metric['namespace']].uuid),
-                "owner": app_map[metric['namespace']].owner_id,
-                "type": "network",
-                "unit": "bytes",
-                "name": "tx",
+                "name": metric['direction'],
                 "usage": value,
                 "kwargs": {
                     "pod": metric['pod'],
diff --git a/rootfs/api/management/commands/upload_volume_usage.py b/rootfs/api/management/commands/upload_volume_usage.py
index 072ca7831..79686c91b 100644
--- a/rootfs/api/management/commands/upload_volume_usage.py
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -21,7 +21,7 @@ def _upload_volume_usage(self, start_time, app_map, timestamp):
         stop = timestamp - (timestamp % 3600)
         start = stop - 3600
         volumes = []
-        for item in async_to_sync(monitor.query_volume_size)(app_map.keys(), start, stop):  # noqa
+        for item in async_to_sync(monitor.query_volume_usage)(app_map.keys(), start, stop):
             metric = item["metric"]
             _, value = item["value"]
             volumes.append({
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 9752aed1d..2d02ca6f3 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -1,54 +1,13 @@
 import time
 import aiohttp
+from string import Template
 from typing import Iterator, AsyncGenerator
 from django.conf import settings
 
 
-query_last_metrics_promql_tpl = """
-last_over_time({__name__=~"%s",namespace="%s"}[%s])
-"""
-
-query_network_receive_flow_promql_tpl = """
-increase(container_network_receive_bytes_total{namespace=~"%s"}[%s])
-"""
-
-query_network_transmit_flow_promql_tpl = """
-increase(container_network_transmit_bytes_total{namespace=~"%s"}[%s])
-"""
-
-query_cpu_usage_promql_tpl = """
-sum (rate (container_cpu_usage_seconds_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
-by (pod)
-"""
-
-
-query_memory_usage_promql_tpl = """
-sum (avg_over_time (container_memory_working_set_bytes{pod=~"^%s-.*$",namespace="%s"}[%s]))
-by (pod)
-"""
-
-query_volume_size_promql_tpl = """
-max by(namespace, persistentvolumeclaim, storageclass) (
-  kube_persistentvolumeclaim_resource_requests_storage_bytes{namespace="%s"}
-  * on(namespace, persistentvolumeclaim) group_left(storageclass)
-  kube_persistentvolumeclaim_info{namespace="%s"}
-  * on(namespace, persistentvolumeclaim) group_left()
-  kube_persistentvolumeclaim_annotations{
-    namespace="%s",
-    annotation_billing_drycc_cc_type="usage"
-  }
-)
-"""
-
-query_network_receive_usage_promql_tpl = """
-sum (rate (container_network_receive_bytes_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
-by (pod)
-"""
-
-query_network_transmit_usage_promql_tpl = """
-sum (rate (container_network_transmit_bytes_total{pod=~"^%s-.*$",namespace="%s"}[%s]))
-by (pod)
-"""
+query_last_metrics_promql_tpl = Template("""
+last_over_time({__name__=~"${metrics}",namespace="${namespace}"}[${duration}])
+""")
 
 
 async def query_prom(url, params) -> list[tuple[dict[str, str], int]]:
@@ -66,10 +25,11 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
     if not settings.DRYCC_METRICS_CONFIG:
         return
     url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = query_last_metrics_promql_tpl % (
-      '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
-      namespace,
-      settings.DRYCC_METRICS_INTERVAL)
+    promql = query_last_metrics_promql_tpl.substitute(
+        metrics='|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
+        namespace=namespace,
+        duration=settings.DRYCC_METRICS_INTERVAL
+    )
     for item in await query_prom(url, {"query": promql, "start": int(time.time() - 60)}):
         yield '%s{%s} %s\n' % (
             item['metric']['__name__'],
@@ -81,59 +41,22 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
         )
 
 
-async def query_volume_size(namespaces: Iterator[str], start: int, stop: int
-                            ) -> list[tuple[dict[str, str], int]]:
-    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = query_volume_size_promql_tpl % (
-        "|".join(namespaces), "|".join(namespaces), "|".join(namespaces))
-    return await query_prom(url, {"query": promql, "start": start, "end": stop})
-
-
-async def query_network_receive_flow(namespaces: Iterator[str], start: int, stop: int
-                                     ) -> list[tuple[dict[str, str], int]]:
+async def query_volume_usage(namespaces: Iterator[str], start: int, stop: int
+                             ) -> list[tuple[dict[str, str], int]]:
+    if not settings.DRYCC_VICTORIAMETRICS_URL:
+        return []
     url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = query_network_receive_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
+    promql = Template(settings.DRYCC_VOLUME_USAGE_TEMPLATE).substitute(
+        namespaces="|".join(namespaces)
+    )
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
 
-async def query_network_transmit_flow(namespaces: Iterator[str], start: int, stop: int
-                                      ) -> list[tuple[dict[str, str], int]]:
+async def query_network_usage(namespaces: Iterator[str], start: int, stop: int
+                              ) -> list[tuple[dict[str, str], int]]:
     url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = query_network_transmit_flow_promql_tpl % ("|".join(namespaces), f"{stop-start}s")
+    promql = Template(settings.DRYCC_NETWORK_USAGE_TEMPLATE).substitute(
+        namespaces="|".join(namespaces),
+        duration=f"{stop-start}s"
+    )
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
-
-
-async def query_cpu_usage(namespace: str, ptype: str, every: str,
-                          start: int, stop: int, step: int,
-                          ) -> list[tuple[dict[str, str], int]]:
-    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
-    pod_prefix = "%s-%s" % (namespace, ptype)
-    promql = query_cpu_usage_promql_tpl % (pod_prefix, namespace, every)
-    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
-
-
-async def query_memory_usage(namespace: str, ptype: str, every: str,
-                             start: int, stop: int, step: int,
-                             ) -> list[tuple[dict[str, str], int]]:
-    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
-    pod_prefix = "%s-%s" % (namespace, ptype)
-    promql = query_memory_usage_promql_tpl % (pod_prefix, namespace, every)
-    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
-
-
-async def query_network_receive_usage(namespace: str, ptype: str, every: str,
-                                      start: int, stop: int, step: int,
-                                      ) -> list[tuple[dict[str, str], int]]:
-    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
-    pod_prefix = "%s-%s" % (namespace, ptype)
-    promql = query_network_receive_usage_promql_tpl % (pod_prefix, namespace, every)
-    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
-
-
-async def query_network_transmit_usage(namespace: str, ptype: str, every: str,
-                                       start: int, stop: int, step: int,
-                                       ) -> list[tuple[dict[str, str], int]]:
-    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query_range"
-    pod_prefix = "%s-%s" % (namespace, ptype)
-    promql = query_network_transmit_usage_promql_tpl % (pod_prefix, namespace, every)
-    return await query_prom(url, {"query": promql, "start": start, "end": stop, "step": step})
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index b686fb253..42e1c4721 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -310,6 +310,22 @@ def randstr(k):
     with open(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH) as fd:
         DRYCC_VOLUME_CLAIM_TEMPLATE = json.load(fd)
 
+# drycc volume usage template
+DRYCC_VOLUME_USAGE_TEMPLATE = ""
+DRYCC_VOLUME_USAGE_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_VOLUME_USAGE_TEMPLATE_PATH', '/etc/controller/volume-usage-template.promql')
+if os.path.exists(DRYCC_VOLUME_USAGE_TEMPLATE_PATH):
+    with open(DRYCC_VOLUME_USAGE_TEMPLATE_PATH) as fd:
+        DRYCC_VOLUME_USAGE_TEMPLATE = fd.read()
+
+# drycc network usage template
+DRYCC_NETWORK_USAGE_TEMPLATE = ""
+DRYCC_NETWORK_USAGE_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_NETWORK_USAGE_TEMPLATE_PATH', '/etc/controller/network-usage-template.promql')
+if os.path.exists(DRYCC_NETWORK_USAGE_TEMPLATE_PATH):
+    with open(DRYCC_NETWORK_USAGE_TEMPLATE_PATH) as fd:
+        DRYCC_NETWORK_USAGE_TEMPLATE = fd.read()
+
 # Django secret key
 SECRET_KEY = os.environ.get('DRYCC_SECRET_KEY', randstr(64))
 

From f7db3e0527462c7382c2c58bacb0fc672ae75ec3 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 5 Sep 2025 17:48:00 +0800
Subject: [PATCH 170/200] chore(usage): add identifier field

---
 .../templates/controller-cronjob-daily.yaml   | 12 ++++++++++
 .../templates/controller-cronjob-hourly.yaml  | 24 +++++++++++++++++++
 .../management/commands/upload_app_usage.py   | 12 ++++++----
 .../commands/upload_gateway_usage.py          | 12 ++++++----
 .../commands/upload_network_usage.py          | 16 ++++++++-----
 .../commands/upload_resource_usage.py         | 12 ++++++----
 .../commands/upload_volume_usage.py           | 16 ++++++++-----
 rootfs/api/models/app.py                      |  1 +
 rootfs/api/models/gateway.py                  |  4 +++-
 rootfs/api/models/resource.py                 |  3 ++-
 10 files changed, 83 insertions(+), 29 deletions(-)

diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index e43456923..6146c9300 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -47,6 +47,10 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.imagePullPolicy}}
             name: drycc-controller-load-db-state-to-k8s
@@ -61,3 +65,11 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          volumes:
+          - name: controller-config
+            configMap:
+              name: controller-config
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index 0a5283440..610cdd803 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -49,6 +49,10 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-upload-gateway-usage
@@ -63,6 +67,10 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-upload-app-usage
@@ -77,6 +85,10 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-upload-resources-usage
@@ -91,6 +103,10 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-upload-volume-usage
@@ -105,3 +121,11 @@ spec:
             {{- end }}
             {{- include "controller.envs" . | indent 12 }}
             {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          volumes:
+          - name: controller-config
+            configMap:
+              name: controller-config
diff --git a/rootfs/api/management/commands/upload_app_usage.py b/rootfs/api/management/commands/upload_app_usage.py
index f0c3aaa80..82028125c 100644
--- a/rootfs/api/management/commands/upload_app_usage.py
+++ b/rootfs/api/management/commands/upload_app_usage.py
@@ -1,5 +1,4 @@
 import uuid
-import time
 import logging
 import random
 from datetime import timedelta
@@ -17,21 +16,24 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} resources to workflow_manager when {now}")
             app_list = []
             for app in App.objects.all():
                 app_list.extend(app.to_usages(timestamp))
                 if len(app_list) % 1000 == 0:
+                    logger.info(f"bulk sent: {len(app_list)}")
                     send_usage.apply_async(
                         args=(app_list,),
-                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                        eta=start_time + timedelta(seconds=random.randint(1, 3600))
                     )
                     app_list = []
             if len(app_list) > 0:
+                logger.info(f"bulk sent: {len(app_list)}")
                 send_usage.apply_async(
                     args=(app_list,),
-                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    eta=start_time + timedelta(seconds=random.randint(1, 3600))
                 )
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_gateway_usage.py b/rootfs/api/management/commands/upload_gateway_usage.py
index 16c6e657a..aaf9b8ba6 100644
--- a/rootfs/api/management/commands/upload_gateway_usage.py
+++ b/rootfs/api/management/commands/upload_gateway_usage.py
@@ -1,5 +1,4 @@
 import uuid
-import time
 import logging
 import random
 from datetime import timedelta
@@ -17,21 +16,24 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} gateways to workflow_manager when {timezone.now()}")
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} gateways to workflow_manager when {now}")
             gateway_list = []
             for gateway in Gateway.objects.all():
                 gateway_list.extend(gateway.to_usages(timestamp))
                 if len(gateway_list) % 1000 == 0:
+                    logger.info(f"bulk sent: {len(gateway_list)}")
                     send_usage.apply_async(
                         args=(gateway_list,),
-                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                        eta=start_time + timedelta(seconds=random.randint(1, 3600))
                     )
                     gateway_list = []
             if len(gateway_list) > 0:
+                logger.info(f"bulk sent: {len(gateway_list)}")
                 send_usage.apply_async(
                     args=(gateway_list,),
-                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    eta=start_time + timedelta(seconds=random.randint(1, 3600))
                 )
             logger.info(f"pushed {task_id} gateways to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_network_usage.py b/rootfs/api/management/commands/upload_network_usage.py
index e82fbe7e0..c24465655 100644
--- a/rootfs/api/management/commands/upload_network_usage.py
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -1,5 +1,5 @@
 import uuid
-import time
+import hashlib
 import logging
 import random
 from datetime import timedelta
@@ -24,6 +24,7 @@ def _upload_network_usage(self, start_time, app_map, timestamp):
         for item in async_to_sync(monitor.query_network_usage)(app_map.keys(), start, stop):
             metric = item["metric"]
             _, value = item["value"]
+            pod_name = metric['pod']
             networks.append({
                 "app_id":  str(app_map[metric['namespace']].uuid),
                 "owner": app_map[metric['namespace']].owner_id,
@@ -32,18 +33,21 @@ def _upload_network_usage(self, start_time, app_map, timestamp):
                 "name": metric['direction'],
                 "usage": value,
                 "kwargs": {
-                    "pod": metric['pod'],
+                    "pod": pod_name,
                 },
-                "timestamp": start
+                "timestamp": start,
+                "identifier": hashlib.md5(pod_name.encode("utf-8")).hexdigest(),
             })
+        logger.info(f"bulk sent: {len(networks)}")
         send_usage.apply_async(
-            args=(networks,), eta=start_time + timedelta(seconds=random.randint(1, 1800))
+            args=(networks,), eta=start_time + timedelta(seconds=random.randint(1, 3600))
         )
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} networks to workflow_manager when {timezone.now()}")
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} networks to workflow_manager when {now}")
             app_map = {}
             for app in App.objects.all():
                 app_map[app.id] = app
diff --git a/rootfs/api/management/commands/upload_resource_usage.py b/rootfs/api/management/commands/upload_resource_usage.py
index b7eb620d3..e9b598912 100644
--- a/rootfs/api/management/commands/upload_resource_usage.py
+++ b/rootfs/api/management/commands/upload_resource_usage.py
@@ -1,5 +1,4 @@
 import uuid
-import time
 import logging
 import random
 from datetime import timedelta
@@ -17,21 +16,24 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            start_time, timestamp, task_id = timezone.now(), time.time(), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} resources to workflow_manager when {now}")
             resource_list = []
             for resource in Resource.objects.filter(status="Ready"):
                 resource_list.extend(resource.to_usages(timestamp))
                 if len(resource_list) % 1000 == 0:
+                    logger.info(f"bulk sent: {len(resource_list)}")
                     send_usage.apply_async(
                         args=(resource_list,),
-                        eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                        eta=start_time + timedelta(seconds=random.randint(1, 3600))
                     )
                     resource_list = []
             if len(resource_list) > 0:
+                logger.info(f"bulk sent: {len(resource_list)}")
                 send_usage.apply_async(
                     args=(resource_list,),
-                    eta=start_time + timedelta(seconds=random.randint(1, 1800))
+                    eta=start_time + timedelta(seconds=random.randint(1, 3600))
                 )
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_volume_usage.py b/rootfs/api/management/commands/upload_volume_usage.py
index 79686c91b..1d4b80e57 100644
--- a/rootfs/api/management/commands/upload_volume_usage.py
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -1,5 +1,5 @@
 import uuid
-import time
+import hashlib
 import logging
 import random
 from datetime import timedelta
@@ -24,6 +24,7 @@ def _upload_volume_usage(self, start_time, app_map, timestamp):
         for item in async_to_sync(monitor.query_volume_usage)(app_map.keys(), start, stop):
             metric = item["metric"]
             _, value = item["value"]
+            pvc_name = metric['persistentvolumeclaim']
             volumes.append({
                 "app_id":  str(app_map[metric['namespace']].uuid),
                 "owner": app_map[metric['namespace']].owner_id,
@@ -32,18 +33,21 @@ def _upload_volume_usage(self, start_time, app_map, timestamp):
                 "name": metric["storageclass"],
                 "usage": value,
                 "kwargs": {
-                    "persistentvolumeclaim": metric['persistentvolumeclaim'],
+                    "persistentvolumeclaim": pvc_name,
                 },
-                "timestamp": start
+                "timestamp": start,
+                "identifier": hashlib.md5(pvc_name.encode("utf-8")).hexdigest(),
             })
+        logger.info(f"bulk sent: {len(volumes)}")
         send_usage.apply_async(
-            args=(volumes,), eta=start_time + timedelta(seconds=random.randint(1, 1800))
+            args=(volumes,), eta=start_time + timedelta(seconds=random.randint(1, 3600))
         )
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            start_time, timestamp, task_id = timezone.now(), int(time.time()), uuid.uuid4().hex
-            logger.info(f"pushing {task_id} volumes to workflow_manager when {timezone.now()}")
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} volumes to workflow_manager when {now}")
             app_map = {}
             for app in App.objects.all():
                 app_map[app.id] = app
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index e286dda62..0fa4b18df 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -741,6 +741,7 @@ def to_usages(self, timestamp: float):
                     "ptype": ptype,
                 },
                 "timestamp": int(timestamp),
+                "identifier": self.uuid.hex,
             })
         return usage
 
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 5c1dcbfe7..4526937be 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -1,4 +1,5 @@
 import logging
+import hashlib
 import threading
 from django.db import models
 from django.conf import settings
@@ -153,7 +154,8 @@ def to_usages(self, timestamp: float):
             "kwargs": {
                 "name": self.name,
             },
-            "timestamp": int(timestamp)
+            "timestamp": int(timestamp),
+            "identifier": hashlib.md5(self.name.encode("utf-8")).hexdigest(),
         }]
 
     def _check_port(self, port, protocol):
diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index a25ca397c..42cdb346f 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -218,7 +218,8 @@ def to_usages(self, timestamp: float):
             "kwargs": {
                 "name": self.name,
             },
-            "timestamp": int(timestamp)
+            "timestamp": int(timestamp),
+            "identifier": self.uuid.hex,
         }]
 
     def _retrieve_status(self):

From 12c2ee2bd9b5349f9f1b845965cfd709e4d2f88f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 8 Sep 2025 08:26:23 +0800
Subject: [PATCH 171/200] chore(tasks): change celery max retries

---
 rootfs/api/tasks.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index 9132d6f93..bd068c6f2 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -15,7 +15,7 @@
     retry_backoff=8,
     retry_jitter=True,
     retry_backoff_max=3600,
-    retry_kwargs={'max_retries': None}
+    retry_kwargs={'max_retries': 32}
 )
 def send_usage(usage: List[Dict[str, str]]):
     task_id = uuid.uuid4().hex
@@ -44,7 +44,7 @@ def send_app_log(app_id, msg, level=logging.INFO):
 
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
-    retry_kwargs={'max_retries': None}
+    retry_kwargs={'max_retries': 3}
 )
 def scale_app(app, user, structure):
     task_id = uuid.uuid4().hex
@@ -60,7 +60,7 @@ def scale_app(app, user, structure):
 
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
-    retry_kwargs={'max_retries': None}
+    retry_kwargs={'max_retries': 3}
 )
 def run_pipeline(release, *args, **kwargs):
     task_id = uuid.uuid4().hex
@@ -162,7 +162,7 @@ def downstream_model_owner(app, old_owner, new_owner):
 
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
-    retry_kwargs={'max_retries': None}
+    retry_kwargs={'max_retries': 3}
 )
 def scale_resources(blocklist, app, suspended_state, scale_type):
     task_id = uuid.uuid4().hex

From 9e2a8273fcfb504d91da2e915137a6b9b06cac2f Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 15 Sep 2025 23:15:34 +0800
Subject: [PATCH 172/200] chore(django): bump version to 5.2.6

---
 rootfs/Dockerfile.test       |  6 +++---
 rootfs/api/models/release.py |  3 +--
 rootfs/api/tests/__init__.py |  4 ++--
 rootfs/dev_requirements.txt  |  6 +++---
 rootfs/requirements.txt      | 34 +++++++++++++++++-----------------
 5 files changed, 26 insertions(+), 27 deletions(-)

diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index f8e6d67fb..f81dded4d 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -8,9 +8,9 @@ COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
 
 ENV PGDATA="/opt/drycc/postgresql/data" \
   PYTHON_VERSION="3.13" \
-  VALKEY_VERSION="8.1.0" \
-  POSTGRES_VERSION="15.12" \
-  GOSU_VERSION="1.17"
+  VALKEY_VERSION="8.1.3" \
+  POSTGRES_VERSION="17.6" \
+  GOSU_VERSION="1.18"
 
 RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
   install-packages mercurial ca-certificates git inotify-tools $buildDeps \
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 687574100..7cf8c750f 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -1,7 +1,6 @@
 import logging
 
-from datetime import datetime
-from django.utils import timezone
+from datetime import datetime, timezone
 from django.conf import settings
 from django.db import models
 from django.db.models import Q
diff --git a/rootfs/api/tests/__init__.py b/rootfs/api/tests/__init__.py
index 23ecade9e..835bb4be1 100644
--- a/rootfs/api/tests/__init__.py
+++ b/rootfs/api/tests/__init__.py
@@ -46,12 +46,12 @@ def fake_responses(request, context):
 class SilentDjangoTestSuiteRunner(DiscoverRunner):
     """Prevents api log messages from cluttering the console during tests."""
 
-    def run_tests(self, test_labels, extra_tests=None, **kwargs):
+    def run_tests(self, test_labels, **kwargs):
         """Run tests with all but critical log messages disabled."""
         # hide any log messages less than critical
         logging.disable(logging.ERROR)
         return super(SilentDjangoTestSuiteRunner, self).run_tests(
-            test_labels, extra_tests, **kwargs)
+            test_labels, **kwargs)
 
 
 class DryccBaseTestCase(unittest.TestCase):
diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt
index 1654cf51a..1f133f1de 100644
--- a/rootfs/dev_requirements.txt
+++ b/rootfs/dev_requirements.txt
@@ -1,13 +1,13 @@
 # test module
 # test
 # Run "make test-unit" for the % of code exercised during tests
-coverage==7.8.0
+coverage==7.10.6
 
 # Run "make test-style" to check python syntax and style
-flake8==7.2.0
+flake8==7.3.0
 
 # mock out python-requests, mostly k8s
 requests-mock==1.12.1
 
 # tracebacks
-tblib==3.1.0
\ No newline at end of file
+tblib==3.1.0
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index d5ea6aa1b..cab3d4afa 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,29 +1,29 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==4.2.22
-channels==4.2.2
-aiohttp==3.12.14
-django-cors-headers==4.7.0
-django-guardian==2.4.0
-djangorestframework==3.16.0
+django==5.2.6
+channels==4.3.1
+aiohttp==3.12.15
+django-cors-headers==4.8.0
+django-guardian==3.1.3
+djangorestframework==3.16.1
 docker==7.1.0
 gunicorn==23.0.0
-uvicorn[standard]==0.34.0
-asgiref==3.8.1
+uvicorn[standard]==0.35.0
+asgiref==3.9.1
 idna==3.10
-jsonschema==4.23.0
+jsonschema==4.25.1
 morph==0.1.5
-packaging==24.2
+packaging==25.0
 pyasn1==0.6.1
-psycopg[binary]==3.2.6
-pyOpenSSL==25.0.0
+psycopg[binary]==3.2.10
+pyOpenSSL==25.1.0
 ndg-httpsclient==0.5.1
 pytz==2025.2
-requests==2.32.4
+requests==2.32.5
 requests-toolbelt==1.0.0
-celery[redis]==5.5.0
+celery[redis]==5.5.3
 hiredis==3.2.1
-dj-database-url==2.3.0
-social-auth-app-django==5.4.3
-kubernetes==32.0.1
+dj-database-url==3.0.1
+social-auth-app-django==5.5.1
+kubernetes==33.1.0

From c2156e1963eb36ba59a2be4b46eeff2779023a96 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 16 Sep 2025 17:19:42 +0800
Subject: [PATCH 173/200] chore(views): change AppResourceViewSet to
 AppFilterViewSet

---
 rootfs/api/views.py | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index de9be4abf..1d3583ab5 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -277,7 +277,7 @@ class BaseDryccViewSet(viewsets.OwnerViewSet):
     renderer_classes = [renderers.JSONRenderer]
 
 
-class AppResourceViewSet(BaseDryccViewSet):
+class AppFilterViewSet(BaseDryccViewSet):
     """A viewset for objects which are attached to an application."""
 
     def get_app(self):
@@ -294,10 +294,10 @@ def get_object(self, **kwargs):
 
     def create(self, request, **kwargs):
         request.data['app'] = self.get_app()
-        return super(AppResourceViewSet, self).create(request, **kwargs)
+        return super(AppFilterViewSet, self).create(request, **kwargs)
 
 
-class ReleasableViewSet(AppResourceViewSet):
+class ReleasableViewSet(AppFilterViewSet):
     """A viewset for application resources which affect the release cycle."""
 
     def get_object(self):
@@ -471,7 +471,7 @@ def post_save(self, config):
             raise DryccException(str(e)) from e
 
 
-class PodViewSet(AppResourceViewSet):
+class PodViewSet(AppFilterViewSet):
     model = models.app.App
     serializer_class = serializers.PodSerializer
 
@@ -499,7 +499,7 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_200_OK)
 
 
-class PtypeViewSet(AppResourceViewSet):
+class PtypeViewSet(AppFilterViewSet):
     model = models.app.App
     serializer_class = serializers.PtypeSerializer
 
@@ -547,7 +547,7 @@ def scale(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class EventViewSet(AppResourceViewSet):
+class EventViewSet(AppFilterViewSet):
     model = models.app.App
     serializer_class = serializers.EventSerializer
 
@@ -566,12 +566,12 @@ def list(self, request, **kwargs):
         return Response(pagination, status=status.HTTP_200_OK)
 
 
-class AppSettingsViewSet(AppResourceViewSet):
+class AppSettingsViewSet(AppFilterViewSet):
     model = models.appsettings.AppSettings
     serializer_class = serializers.AppSettingsSerializer
 
 
-class DomainViewSet(AppResourceViewSet):
+class DomainViewSet(AppFilterViewSet):
     """A viewset for interacting with Domain objects."""
     model = models.domain.Domain
     serializer_class = serializers.DomainSerializer
@@ -596,7 +596,7 @@ def destroy(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class ServiceViewSet(AppResourceViewSet):
+class ServiceViewSet(AppFilterViewSet):
     """A viewset for interacting with Service objects."""
     model = models.service.Service
     serializer_class = serializers.ServiceSerializer
@@ -640,7 +640,7 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class CertificateViewSet(AppResourceViewSet):
+class CertificateViewSet(AppFilterViewSet):
     """A viewset for interacting with Certificate objects."""
     model = models.certificate.Certificate
     serializer_class = serializers.CertificateSerializer
@@ -678,7 +678,7 @@ class KeyViewSet(BaseDryccViewSet):
     serializer_class = serializers.KeySerializer
 
 
-class ReleaseViewSet(AppResourceViewSet):
+class ReleaseViewSet(AppFilterViewSet):
     """A viewset for interacting with Release objects."""
     model = models.release.Release
     serializer_class = serializers.ReleaseSerializer
@@ -729,7 +729,7 @@ def rollback(self, request, **kwargs):
         return Response(response, status=status.HTTP_201_CREATED)
 
 
-class TLSViewSet(AppResourceViewSet):
+class TLSViewSet(AppFilterViewSet):
     model = models.tls.TLS
     serializer_class = serializers.TLSSerializer
 
@@ -858,7 +858,7 @@ def create(self, request, *args, **kwargs):
         return Response(serializer.data, status=status.HTTP_200_OK)
 
 
-class AppPermViewSet(AppResourceViewSet):
+class AppPermViewSet(AppFilterViewSet):
     """RESTful views for sharing apps with collaborators."""
 
     def get_app(self, request):
@@ -934,7 +934,7 @@ def destroy(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class AppVolumesViewSet(AppResourceViewSet):
+class AppVolumesViewSet(AppFilterViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
     serializer_class = serializers.VolumeSerializer
@@ -985,7 +985,7 @@ def path(self, request, *args, **kwargs):
         return Response(serializer.data)
 
 
-class AppFilerClientViewSet(AppResourceViewSet):
+class AppFilerClientViewSet(AppFilterViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
     parser_classes = [FilerUploadParser]
@@ -1030,7 +1030,7 @@ def destroy(self, request, **kwargs):
         return Response(data=response.content, status=response.status_code)
 
 
-class AppResourcesViewSet(AppResourceViewSet):
+class AppResourcesViewSet(AppFilterViewSet):
     """RESTful views for resources apps with collaborators."""
     model = models.resource.Resource
     serializer_class = serializers.ResourceSerializer
@@ -1053,7 +1053,7 @@ def plans(self, request, *args, **kwargs):
         ))
 
 
-class AppSingleResourceViewSet(AppResourceViewSet):
+class AppSingleResourceViewSet(AppFilterViewSet):
     """RESTful views for resource apps with collaborators."""
     model = models.resource.Resource
     serializer_class = serializers.ResourceSerializer
@@ -1085,7 +1085,7 @@ def update(self, request, *args, **kwargs):
         return Response(resource.data)
 
 
-class AppResourceBindingViewSet(AppResourceViewSet):
+class AppResourceBindingViewSet(AppFilterViewSet):
     model = models.resource.Resource
     serializer_class = serializers.ResourceSerializer
 
@@ -1112,7 +1112,7 @@ def binding(self, request, *args, **kwargs):
             return Response("unknown action", status=status.HTTP_404_NOT_FOUND)
 
 
-class GatewayViewSet(AppResourceViewSet):
+class GatewayViewSet(AppFilterViewSet):
     """A viewset for interacting with Gateway objects."""
     model = models.gateway.Gateway
     filter_backends = [filters.SearchFilter]
@@ -1148,7 +1148,7 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class RouteViewSet(AppResourceViewSet):
+class RouteViewSet(AppFilterViewSet):
     """A viewset for interacting with Route objects."""
     model = models.gateway.Route
     filter_backends = [filters.SearchFilter]

From 8a8843b6b94830a9d534abb0c0d1d7f2a918a1a3 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Wed, 24 Sep 2025 16:07:08 +0800
Subject: [PATCH 174/200] chore(controller): update release state when force
 deploy (#194)

* chore(controller): update release state when force deploy

* chore(controller): frequent cert detach may lead to gateway patch 409 sometime

* chore(controller): config detach unexpected ptype
---
 rootfs/api/models/app.py                 | 2 +-
 rootfs/api/models/gateway.py             | 6 +++++-
 rootfs/api/views.py                      | 7 ++++---
 rootfs/scheduler/resources/deployment.py | 4 ++--
 rootfs/scheduler/resources/gateway.py    | 6 ++++--
 5 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 0fa4b18df..d1bf19f68 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -312,7 +312,7 @@ def pipeline(self, release, ptypes, force_deploy=False):
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
                 self.deploy(release, ptypes, force_deploy, rollback_on_failure)
-            if release.state == "created":
+            if release.state in ["created", "crashed"]:
                 release.state = "succeed"
             ptypes = list(ptypes) if ptypes is not None else ptypes
             release.add_condition(state="succeed", action="pipeline", ptypes=ptypes)
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 4526937be..71dfef91b 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -105,13 +105,17 @@ def refresh_to_k8s(self):
                 data = self.scheduler.gateways.get(self.app.id, self.name).json()
                 if len(kwargs["listeners"]) > 0:
                     kwargs["version"] = data["metadata"]["resourceVersion"]
-                    self.scheduler.gateways.patch(self.app.id, self.name, **kwargs)
+                    response = self.scheduler.gateways.patch(self.app.id, self.name, **kwargs)
+                    if response.status_code == 409:
+                        raise ServiceUnavailable(f'Kubernetes gateway could not be patched: {response.status_code} {response.reason}, please retry')  # noqa 
                 else:
                     logger.debug("delete k8s resource when listeners are empty")
                     self.scheduler.gateways.delete(
                         self.app.id, self.name, ignore_exception=True)
             except KubeException:
                 if len(kwargs["listeners"]) > 0:
+                    if "version" in kwargs:
+                        kwargs.pop("version")
                     self.scheduler.gateways.create(self.app.id, self.name, **kwargs)
                 else:
                     logger.debug("skip creating k8s resource when listeners are empty")
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 1d3583ab5..38804bd27 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -445,9 +445,10 @@ def delete(self, request, **kwargs):
 
         config = self.model(app=self.get_app(), owner=self.request.user, values_refs={})
         old_values_refs = config.previous().values_refs.copy()
-        for ptype, groups in values_refs.items():
-            for group in old_values_refs.get(ptype, []):
-                if group not in groups:
+        for ptype, old_groups in old_values_refs.items():
+            groups_to_delete = values_refs.get(ptype, [])
+            for group in old_groups:
+                if group not in groups_to_delete:
                     if ptype not in config.values_refs:
                         config.values_refs[ptype] = [group]
                     elif group not in config.values_refs[ptype]:
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index bf9485553..0a8b2bae9 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -284,7 +284,7 @@ def in_progress(self, namespace, name, timeout, batches, replicas, tags):
                 namespace,
                 'Deployment {} had stalled due an error and will be rolled back. {}'.format(
                     name, str(e)),
-                level='DEBUG'
+                level=logging.DEBUG
             )
             return False, True
 
@@ -305,7 +305,7 @@ def in_progress(self, namespace, name, timeout, batches, replicas, tags):
                 namespace,
                 'Deploy operation for Deployment {} in has expired. '
                 'Rolling back to last good known release'.format(name),
-                level='DEBUG',
+                level=logging.DEBUG,
             )
             return False, True
 
diff --git a/rootfs/scheduler/resources/gateway.py b/rootfs/scheduler/resources/gateway.py
index 931bd8c86..a51ff538a 100644
--- a/rootfs/scheduler/resources/gateway.py
+++ b/rootfs/scheduler/resources/gateway.py
@@ -59,8 +59,10 @@ def patch(self, namespace, name, ignore_exception=False, **kwargs):
             json=data,
             headers={"Content-Type": "application/merge-patch+json"}
         )
+        if response.status_code == 409:
+            return response
         if not ignore_exception and self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, "put gateway {}".format(namespace))
+            raise KubeHTTPException(response, "patch gateway {}".format(namespace))
         return response
 
     def delete(self, namespace, name, ignore_exception=True):
@@ -128,7 +130,7 @@ def patch(self, namespace, name, ignore_exception=False, **kwargs):
             headers={"Content-Type": "application/merge-patch+json"}
         )
         if not ignore_exception and self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, "put {} {}".format(self.kind.lower(), namespace))
+            raise KubeHTTPException(response, "patch {} {}".format(self.kind.lower(), namespace))
         return response
 
     def delete(self, namespace, name, ignore_exception=True):

From e88ba9989eb70a1f6cd55877640a3166e707e141 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 25 Sep 2025 11:48:25 +0800
Subject: [PATCH 175/200] chore(config): add merge params for create api

---
 rootfs/api/tests/test_config.py | 72 +++++++++++++++++++++++++++++++++
 rootfs/api/views.py             | 23 +++++++----
 2 files changed, 88 insertions(+), 7 deletions(-)

diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index f24e0054a..9a3508bfe 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -141,6 +141,78 @@ def test_config(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         return config5
 
+    def test_config_merge_false(self, mock_requests):
+        """
+        Test that config can be created with merge='false' parameter
+        which replaces the entire config instead of merging
+        """
+        app_id = self.create_app()
+
+        # set initial config values
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "group": "global"}
+        value2 = {"name": "DATABASE_URL", "value": "postgres://localhost/db", "group": "global"}
+        body = {'values': [value1, value2]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(len(response.data['values']), 2)
+
+        # replace entire config with merge='false'
+        value3 = {"name": "NEW_CONFIG", "value": "replacement_value", "group": "global"}
+        body = {'values': [value3]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # verify only the new config exists
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 1)
+        self.assertEqual(response.data['values'][0]['name'], "NEW_CONFIG")
+        self.assertEqual(response.data['values'][0]['value'], "replacement_value")
+
+    def test_config_merge_false_with_empty_values(self, mock_requests):
+        """
+        Test that config can be created with merge='false' and empty values
+        """
+        app_id = self.create_app()
+
+        # set initial config values
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "group": "global"}
+        body = {'values': [value1]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(len(response.data['values']), 1)
+
+        # replace with empty config using merge='false'
+        body = {'values': []}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # verify config is empty
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 0)
+
+    def test_config_merge_false_case_insensitive(self, mock_requests):
+        """
+        Test that merge parameter is case insensitive
+        """
+        app_id = self.create_app()
+
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "group": "global"}
+        body = {'values': [value1]}
+
+        # test with uppercase FALSE
+        response = self.client.post(f"{url}?merge=FALSE", body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # verify config was replaced
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 1)
+
     def test_registry_set(self, mock_requests):
         app_id = self.create_app()
         # set an initial config value
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 38804bd27..1ff0be89f 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -260,7 +260,7 @@ def get_queryset(self):
 
     def destroy(self, *args, **kwargs):
         key = self.get_object().key
-        response = super(TokenViewSet, self).destroy(self, *args, **kwargs)
+        response = super().destroy(self, *args, **kwargs)
         cache.delete(key)
         return response
 
@@ -294,7 +294,7 @@ def get_object(self, **kwargs):
 
     def create(self, request, **kwargs):
         request.data['app'] = self.get_app()
-        return super(AppFilterViewSet, self).create(request, **kwargs)
+        return super().create(request, **kwargs)
 
 
 class ReleasableViewSet(AppFilterViewSet):
@@ -327,7 +327,7 @@ def list(self, request, *args, **kwargs):
         which are owned by the user as well as any apps they have been given permission to
         interact with.
         """
-        queryset = super(AppViewSet, self).get_queryset(**kwargs) | \
+        queryset = super().get_queryset(**kwargs) | \
             get_objects_for_user(
                 self.request.user, f'api.{models.app.VIEW_APP_PERMISSION.codename}')
         instance = self.filter_queryset(queryset)
@@ -394,7 +394,7 @@ def post_save(self, build):
             if is_loopback(image):
                 raise DryccException("image must not use the loopback address")
         build.create_release(self.request.user)
-        super(BuildViewSet, self).post_save(build)
+        super().post_save(build)
 
 
 class LimitSpecViewSet(BaseDryccViewSet):
@@ -438,6 +438,15 @@ class ConfigViewSet(ReleasableViewSet):
     model = models.config.Config
     serializer_class = serializers.ConfigSerializer
 
+    def create(self, request, **kwargs):
+        if self.request.query_params.get('merge', 'true').lower() == 'true':
+            return super().create(request, **kwargs)
+        values = self.get_serializer().validate_values(request.data.get('values'))
+        config = self.model(app=self.get_app(), owner=self.request.user, values=values)
+        config.save(ignore_update_fields=["values"])
+        self.post_save(config)
+        return Response(status=status.HTTP_201_CREATED)
+
     def delete(self, request, **kwargs):
         values_refs = self.get_serializer().validate_values_refs(request.data.get('values_refs'))
         if not values_refs or not values_refs.values():
@@ -691,7 +700,7 @@ def get_object(self, **kwargs):
 
     def get_queryset(self, **kwargs):
         ptypes = self.request.query_params.get('ptypes', '').strip()
-        queryset = super(ReleaseViewSet, self).get_queryset(**kwargs)
+        queryset = super().get_queryset(**kwargs)
         if ptypes:
             queryset = queryset.filter(Q(
                 deployed_ptypes__contains=[
@@ -829,7 +838,7 @@ def create(self, request, *args, **kwargs):
             return Response(message, status=status.HTTP_403_FORBIDDEN)
         request.data['app'] = app
         request.data['owner'] = self.user
-        super(BuildHookViewSet, self).create(request, *args, **kwargs)
+        super().create(request, *args, **kwargs)
         # return the application databag
         response = {
             'release': {
@@ -1067,7 +1076,7 @@ def get_object(self):
     def retrieve(self, request, *args, **kwargs):
         resource = self.get_object()
         resource.retrieve(request)
-        response =  super(AppSingleResourceViewSet, self).retrieve(request, *args, **kwargs)  # noqa
+        response =  super().retrieve(request, *args, **kwargs)  # noqa
         response.data["message"] = resource.message
         return response
 

From 7bb9336ffe630bcde0ee53414bc32248257744e6 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 25 Sep 2025 13:55:02 +0800
Subject: [PATCH 176/200] fix(dryccpath): enforce replace mode for env
 variables

---
 rootfs/api/models/build.py      | 11 ++++++-----
 rootfs/api/models/config.py     |  8 ++++----
 rootfs/api/tests/test_config.py | 12 ++++++------
 rootfs/api/views.py             |  5 +++++
 4 files changed, 21 insertions(+), 15 deletions(-)

diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 5b2060ee8..41da3ddc2 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -120,13 +120,14 @@ def _get_or_create_config(self):
         """
         dryccfile to config
         """
-        config_values, config_values_ref, changed_fields = [], {}, set()
+        config_values, config_values_ref, changed_fields, replace_groups = [], {}, set(), set()
         for group, envs in self.dryccfile.get('config', {}).items():
             for key, value in envs.items():
+                replace_groups.add(group)
                 config_values.append({"name": key, "group": group, "value": value})
             changed_fields.update(["values", "values_refs"])
 
-        replace_values_ptypes = set()
+        replace_ptypes = set()
         for pipeline in self.dryccfile.get('pipeline', {}).values():
             if 'env' in pipeline or 'config' in pipeline:
                 for key, value in pipeline.get('env', {}).items():
@@ -136,7 +137,7 @@ def _get_or_create_config(self):
                         config_values_ref[pipeline['ptype']] = [config_ref]
                     else:
                         config_values_ref[pipeline['ptype']].append(config_ref)
-                replace_values_ptypes.add(pipeline['ptype'])
+                replace_ptypes.add(pipeline['ptype'])
                 changed_fields.update(["values", "values_refs"])
 
         old_config = self.app.release_set.filter(failed=False).latest().config
@@ -144,7 +145,7 @@ def _get_or_create_config(self):
             return old_config
         config = Config(
             owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref)
-        config.merge_field("values", old_config, replace_values_ptypes)
-        config.merge_field("values_refs", old_config, replace_values_ptypes)
+        config.merge_field("values", old_config, replace_ptypes, replace_groups)
+        config.merge_field("values_refs", old_config, replace_ptypes)
         config.save(ignore_update_fields=changed_fields)
         return config
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 1f7ba80c6..b298a60fd 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -139,12 +139,12 @@ def save(self, ignore_update_fields=None, *args, **kwargs):
             self._update_tags(previous_config={'tags': {}})
         return super(Config, self).save(*args, **kwargs)
 
-    def merge_field(self, field, old_config, replace_ptypes=[]):
+    def merge_field(self, field, old_config, *args, **kwargs):
         getattr(
             self,
             "_update_%s" % field,
             partial(self._update_field, field)
-        )(old_config, replace_ptypes)
+        )(old_config, *args, **kwargs)
 
     def _update_field(self, field, previous_config, replace_ptypes=[]):
         data = {
@@ -164,10 +164,10 @@ def _update_field(self, field, previous_config, replace_ptypes=[]):
                 data[key] = self._merge_data(field, data.get(key, {}), value)
         setattr(self, field, data)
 
-    def _update_values(self, previous_config, replace_ptypes=[]):
+    def _update_values(self, previous_config, replace_ptypes=[], replace_groups=[]):
         data = [
             item for item in getattr(previous_config, 'values', []).copy()
-            if item.get('ptype') not in replace_ptypes
+            if item.get('ptype') not in replace_ptypes and item.get('group') not in replace_groups
         ]
         new_data = getattr(self, 'values', []).copy()
         for new_item in new_data:
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 9a3508bfe..5ff9d2f0a 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -187,12 +187,12 @@ def test_config_merge_false_with_empty_values(self, mock_requests):
         # replace with empty config using merge='false'
         body = {'values': []}
         response = self.client.post(f"{url}?merge=false", body)
-        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.status_code, 409, response.data)
 
         # verify config is empty
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['values']), 0)
+        self.assertEqual(len(response.data['values']), 1)
 
     def test_config_merge_false_case_insensitive(self, mock_requests):
         """
@@ -970,9 +970,9 @@ def test_config_from_dryccfile(self, mock_requests):
         self.assertEqual(
             release.config.envs("web"),
             {
-                'GROUP': 'g1', 'DEBUG': 'tr', 'TEST1': 'g1', 'TEST2': 'tr', 'PENV1': 'web',
-                'PENV2': 'web', "WEBSITE": "www.drycc.cc",
-            }
+                'WEBSITE': 'www.drycc.cc', 'GROUP': 'g1', 'DEBUG': 'tr',
+                'PENV1': 'web', 'PENV2': 'web',  'TEST1': 'g1', 'TEST2': 'tr',
+            },
         )
 
         # test use old config and healthcheck
@@ -1001,5 +1001,5 @@ def test_config_from_dryccfile(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.failed, False)
-        self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
+        self.assertEqual(release.config.envs("web"), {'WEBSITE': 'www.drycc.cc'})
         self.assertEqual(release.config.values_refs, {})
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 1ff0be89f..2694a6fba 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -443,6 +443,11 @@ def create(self, request, **kwargs):
             return super().create(request, **kwargs)
         values = self.get_serializer().validate_values(request.data.get('values'))
         config = self.model(app=self.get_app(), owner=self.request.user, values=values)
+        old_config = config.previous()
+        if old_config and old_config.values:
+            replace_ptypes = {v['ptype'] for v in config.values if 'ptype' in v}
+            replace_groups = {v['group'] for v in config.values if 'group' in v}
+            config.merge_field("values", old_config, replace_ptypes, replace_groups)
         config.save(ignore_update_fields=["values"])
         self.post_save(config)
         return Response(status=status.HTTP_201_CREATED)

From a55d157ab6fa05acdd156479dd3be7cc7f14d450 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 30 Sep 2025 11:26:59 +0800
Subject: [PATCH 177/200] fix(metrics): metrics proxy error

---
 charts/controller/templates/_helpers.tpl |  4 +++
 rootfs/api/tests/test_config.py          | 43 ++++++++++++++++++++----
 rootfs/api/views.py                      |  4 ++-
 3 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 940231ef5..56dbc8da8 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -346,6 +346,10 @@ kubelet_volume_stats_capacity_bytes: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim, job]
 kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim, job]
+istio_requests_total: [source_workload_namespace, reporter, destination_service, response_code, connection_security_policy, source_workload]
+istio_request_duration_milliseconds_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
+istio_request_bytes_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
+istio_response_bytes_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
 {{- end }}
 
 {{/* Generate controller config default reserved name patterns */}}
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 5ff9d2f0a..3d310cf0d 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -141,7 +141,7 @@ def test_config(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         return config5
 
-    def test_config_merge_false(self, mock_requests):
+    def test_config_group_merge_false(self, mock_requests):
         """
         Test that config can be created with merge='false' parameter
         which replaces the entire config instead of merging
@@ -159,16 +159,47 @@ def test_config_merge_false(self, mock_requests):
 
         # replace entire config with merge='false'
         value3 = {"name": "NEW_CONFIG", "value": "replacement_value", "group": "global"}
-        body = {'values': [value3]}
+        body = {'values': [value1, value2, value3]}
         response = self.client.post(f"{url}?merge=false", body)
         self.assertEqual(response.status_code, 201, response.data)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 3)
 
-        # verify only the new config exists
+        # remove value3
+        body = {'values': [value1, value2]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['values']), 1)
-        self.assertEqual(response.data['values'][0]['name'], "NEW_CONFIG")
-        self.assertEqual(response.data['values'][0]['value'], "replacement_value")
+        self.assertEqual(len(response.data['values']), 2)
+
+    def test_config_ptype_merge_false(self, mock_requests):
+        """
+        Test that config can be created with merge='false' parameter
+        which replaces the entire config instead of merging
+        """
+        app_id = self.create_app()
+
+        # set initial config values
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "ptype": "web"}
+        value2 = {"name": "DATABASE_URL", "value": "postgres://localhost/db", "ptype": "web"}
+        value3 = {"name": "NEW_CONFIG", "value": "replacement_value", "ptype": "web"}
+        body = {'values': [value1, value2, value3]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 3)
+
+        # remove value3
+        body = {'values': [value1, value2]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 2)
 
     def test_config_merge_false_with_empty_values(self, mock_requests):
         """
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 2694a6fba..a0e37240c 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1286,10 +1286,12 @@ async def sample(self, name, labels_str, value):
         fields = set(settings.DRYCC_METRICS_CONFIG[name])
         labels = {}
         for pair in labels_str.strip(" {}").split(','):
+            if '=' not in pair:
+                continue
             k, v = pair.split('=', 1)
             if k in fields:
                 labels[k] = v.strip(' "')
-        app_id = labels.get("namespace", None)
+        app_id = labels.get("namespace", labels.get(settings.DRYCC_METRICS_CONFIG[name][0]))
         if not app_id:
             return None
         account_id, timeout = self.cache.get(app_id, self.default_cache_value)

From 8fb613d0ce0a4909319a6d08b85e669cf3db676e Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 10 Oct 2025 15:49:55 +0800
Subject: [PATCH 178/200] fix(config): merge=false no data return

---
 rootfs/api/views.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index a0e37240c..ecfa8b4f8 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -450,7 +450,9 @@ def create(self, request, **kwargs):
             config.merge_field("values", old_config, replace_ptypes, replace_groups)
         config.save(ignore_update_fields=["values"])
         self.post_save(config)
-        return Response(status=status.HTTP_201_CREATED)
+        data = self.get_serializer(config).data
+        headers = self.get_success_headers(data)
+        return Response(data, status=status.HTTP_201_CREATED, headers=headers)
 
     def delete(self, request, **kwargs):
         values_refs = self.get_serializer().validate_values_refs(request.data.get('values_refs'))

From 48be01dd42d19a82112404d47a785f7f9a8d0660 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 10 Oct 2025 15:50:44 +0800
Subject: [PATCH 179/200] chore(deps): bump social-auth-app-django from 5.5.1
 to 5.6.0 in /rootfs (#195)

Bumps [social-auth-app-django](https://github.com/python-social-auth/social-app-django) from 5.5.1 to 5.6.0.
- [Release notes](https://github.com/python-social-auth/social-app-django/releases)
- [Changelog](https://github.com/python-social-auth/social-app-django/blob/master/CHANGELOG.md)
- [Commits](https://github.com/python-social-auth/social-app-django/compare/5.5.1...5.6.0)

---
updated-dependencies:
- dependency-name: social-auth-app-django
  dependency-version: 5.6.0
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index cab3d4afa..98f17752e 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -25,5 +25,5 @@ requests-toolbelt==1.0.0
 celery[redis]==5.5.3
 hiredis==3.2.1
 dj-database-url==3.0.1
-social-auth-app-django==5.5.1
+social-auth-app-django==5.6.0
 kubernetes==33.1.0

From 9ffb6fef8c04dff73e99a8aa609adf6a5e6c0f96 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 10 Oct 2025 15:51:41 +0800
Subject: [PATCH 180/200] chore(deps): bump django from 5.2.6 to 5.2.7 in
 /rootfs (#196)

Bumps [django](https://github.com/django/django) from 5.2.6 to 5.2.7.
- [Commits](https://github.com/django/django/compare/5.2.6...5.2.7)

---
updated-dependencies:
- dependency-name: django
  dependency-version: 5.2.7
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 98f17752e..22e689410 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==5.2.6
+django==5.2.7
 channels==4.3.1
 aiohttp==3.12.15
 django-cors-headers==4.8.0

From d4dbfd407eaf54cfe7e3f831dab32d661062cc4f Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Sat, 11 Oct 2025 17:11:57 +0800
Subject: [PATCH 181/200] fix(pipeline): state and failed fields update timing
 (#197)

---
 rootfs/api/models/app.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index d1bf19f68..c58b61f15 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -312,8 +312,8 @@ def pipeline(self, release, ptypes, force_deploy=False):
                 if not rollback_on_failure:
                     self.log(f"{prefix} deploy do not rollback on failure")
                 self.deploy(release, ptypes, force_deploy, rollback_on_failure)
-            if release.state in ["created", "crashed"]:
-                release.state = "succeed"
+
+            release.state, release.failed = "succeed", False
             ptypes = list(ptypes) if ptypes is not None else ptypes
             release.add_condition(state="succeed", action="pipeline", ptypes=ptypes)
         except Exception as e:

From e9937da2dfa9644fecb3868fb5d65564d0473930 Mon Sep 17 00:00:00 2001
From: Eamon <zhang.eamon@hotmail.com>
Date: Tue, 14 Oct 2025 10:43:03 +0800
Subject: [PATCH 182/200] chore(controller):modify controller config default
 metrics (#198)

* chore(controller):modify controller config default metrics

* megre(controller): upstream

---------

Co-authored-by: zhangeamon <EamonZhang>
---
 charts/controller/templates/_helpers.tpl | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 56dbc8da8..a9cb0ea73 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -340,12 +340,12 @@ kube_pod_container_status_terminated: [container, namespace, node, pod]
 kube_pod_container_status_waiting: [container, namespace, node, pod]
 kube_pod_container_status_restarts_total: [container, namespace, node, pod]
 kube_pod_spec_volumes_persistentvolumeclaims_info: [namespace, pod, persistentvolumeclaim]
-kubelet_volume_stats_used_bytes: [namespace, persistentvolumeclaim, job]
-kubelet_volume_stats_available_bytes: [namespace, persistentvolumeclaim, job]
-kubelet_volume_stats_capacity_bytes: [namespace, persistentvolumeclaim, job]
-kubelet_volume_stats_inodes: [namespace, persistentvolumeclaim, job]
-kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim, job]
-kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim, job]
+kubelet_volume_stats_used_bytes: [namespace, persistentvolumeclaim]
+kubelet_volume_stats_available_bytes: [namespace, persistentvolumeclaim]
+kubelet_volume_stats_capacity_bytes: [namespace, persistentvolumeclaim]
+kubelet_volume_stats_inodes: [namespace, persistentvolumeclaim]
+kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim]
+kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim]
 istio_requests_total: [source_workload_namespace, reporter, destination_service, response_code, connection_security_policy, source_workload]
 istio_request_duration_milliseconds_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
 istio_request_bytes_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]

From 221532723a3bada1517e93e27ebe70fd4d5d3505 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Wed, 15 Oct 2025 09:30:58 +0800
Subject: [PATCH 183/200] chore(charts): use Files.Get replace include

---
 charts/controller/files/limit-plans.json      | 982 ++++++++++++++++++
 charts/controller/files/limit-specs.json      |  38 +
 charts/controller/files/metrics.json          | 466 +++++++++
 .../files/network-usage-template.promql       |   9 +
 .../files/reserved-name-patterns.txt          |  12 +
 .../files/volume-claim-template.json          |  61 ++
 charts/controller/files/volume-template.json  |  55 +
 .../files/volume-usage-template.promql        |  12 +
 charts/controller/templates/_helpers.tpl      | 288 -----
 .../templates/controller-configmap.yaml       |  22 +-
 rootfs/api/models/volume.py                   |  55 +-
 rootfs/api/monitor.py                         |  28 +-
 rootfs/api/settings/production.py             |  12 +-
 rootfs/api/settings/testing.py                |  87 +-
 rootfs/api/tests/test_volume.py               |   3 +-
 15 files changed, 1733 insertions(+), 397 deletions(-)
 create mode 100644 charts/controller/files/limit-plans.json
 create mode 100644 charts/controller/files/limit-specs.json
 create mode 100644 charts/controller/files/metrics.json
 create mode 100644 charts/controller/files/network-usage-template.promql
 create mode 100644 charts/controller/files/reserved-name-patterns.txt
 create mode 100644 charts/controller/files/volume-claim-template.json
 create mode 100644 charts/controller/files/volume-template.json
 create mode 100644 charts/controller/files/volume-usage-template.promql

diff --git a/charts/controller/files/limit-plans.json b/charts/controller/files/limit-plans.json
new file mode 100644
index 000000000..2b9a7fe29
--- /dev/null
+++ b/charts/controller/files/limit-plans.json
@@ -0,0 +1,982 @@
+[
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "1Gi"
+      },
+      "memory": 1,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10100,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "0.5Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m1"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "2Gi"
+      },
+      "memory": 2,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10200,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "1Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m2"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "4Gi"
+      },
+      "memory": 4,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10300,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "2Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m4"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10400,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "2Gi"
+      },
+      "memory": 2,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10500,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "1Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m2"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "4Gi"
+      },
+      "memory": 4,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10600,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "2Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m4"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10700,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10800,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "4Gi"
+      },
+      "memory": 4,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10900,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "2Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m4"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11000,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11100,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11200,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11300,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11400,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11500,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11600,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11700,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11800,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11900,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "128Gi"
+      },
+      "memory": 128,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12000,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "64Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m128"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12100,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12200,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "128Gi"
+      },
+      "memory": 128,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12300,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "64Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m128"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "256Gi"
+      },
+      "memory": 256,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12400,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "128Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m256"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12500,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "128Gi"
+      },
+      "memory": 128,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12600,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "64Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m128"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "256Gi"
+      },
+      "memory": 256,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12700,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "128Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m256"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "512Gi"
+      },
+      "memory": 512,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12800,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "256Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m512"
+  }
+]
diff --git a/charts/controller/files/limit-specs.json b/charts/controller/files/limit-specs.json
new file mode 100644
index 000000000..732a70e2d
--- /dev/null
+++ b/charts/controller/files/limit-specs.json
@@ -0,0 +1,38 @@
+[
+  {
+    "fields": {
+      "cpu": {
+        "boost": "3700MHZ",
+        "clock": "3100MHZ",
+        "cores": 32,
+        "name": "Universal CPU",
+        "threads": 64
+      },
+      "created": "2025-09-16T05:05:37.135Z",
+      "disabled": false,
+      "features": {
+        "gpu": {
+          "cores": 128,
+          "memory": {
+            "size": "shared",
+            "type": "shared"
+          },
+          "name": "Integrated GPU",
+          "rops": 1,
+          "tmus": 1
+        },
+        "network": "10G"
+      },
+      "keywords": [
+        "unknown"
+      ],
+      "memory": {
+        "size": "64GB",
+        "type": "DDR4-ECC"
+      },
+      "updated": "2025-09-16T05:05:37.135Z"
+    },
+    "model": "api.limitspec",
+    "pk": "std1"
+  }
+]
diff --git a/charts/controller/files/metrics.json b/charts/controller/files/metrics.json
new file mode 100644
index 000000000..e620eeec2
--- /dev/null
+++ b/charts/controller/files/metrics.json
@@ -0,0 +1,466 @@
+{
+  "container_cpu_load_average_10s": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_system_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_usage_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_user_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_cfs_periods_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_cfs_throttled_periods_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_cfs_throttled_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_inodes_free": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_usage_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_inodes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_io_current": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_io_time_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_io_time_weighted_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_reads_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_read_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_reads_merged_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_reads_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_sector_reads_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_sector_writes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_writes_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_write_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_writes_merged_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_writes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_blkio_device_usage_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_failures_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_failcnt": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_cache": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_mapped_file": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_max_usage_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_rss": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_swap": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_usage_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_working_set_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_errors_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_packets_dropped_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_packets_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_errors_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_packets_dropped_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_packets_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_oom_events_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_processes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_sockets": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_file_descriptors": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_threads": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_threads_max": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_ulimits_soft": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_cpu_quota": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_cpu_period": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_cpu_shares": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_memory_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_memory_reservation_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_memory_swap_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_start_time_seconds": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_tasks_state": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_last_seen": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_accelerator_memory_used_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_accelerator_memory_total_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_accelerator_duty_cycle": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "kube_pod_ips": [
+    "ip",
+    "ip_family",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_running": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_ready": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_terminated": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_waiting": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_restarts_total": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_spec_volumes_persistentvolumeclaims_info": [
+    "namespace",
+    "pod",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_used_bytes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_available_bytes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_capacity_bytes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_inodes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_inodes_free": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_inodes_used": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "istio_requests_total": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "response_code",
+    "connection_security_policy",
+    "source_workload"
+  ],
+  "istio_request_duration_milliseconds_bucket": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "connection_security_policy",
+    "source_workload",
+    "le"
+  ],
+  "istio_request_bytes_bucket": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "connection_security_policy",
+    "source_workload",
+    "le"
+  ],
+  "istio_response_bytes_bucket": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "connection_security_policy",
+    "source_workload",
+    "le"
+  ]
+}
\ No newline at end of file
diff --git a/charts/controller/files/network-usage-template.promql b/charts/controller/files/network-usage-template.promql
new file mode 100644
index 000000000..07a608de1
--- /dev/null
+++ b/charts/controller/files/network-usage-template.promql
@@ -0,0 +1,9 @@
+label_replace(
+  sum(increase(istio_request_bytes_sum{namespace=~"{{namespaces}}"}[{{duration}}])) by (namespace,pod),
+  "direction", "rx", "", ""
+)
+or
+label_replace(
+  sum(increase(istio_response_bytes_sum{namespace=~"{{namespaces}}"}[{{duration}}])) by (namespace,pod),
+  "direction", "tx", "", ""
+)
diff --git a/charts/controller/files/reserved-name-patterns.txt b/charts/controller/files/reserved-name-patterns.txt
new file mode 100644
index 000000000..71574be55
--- /dev/null
+++ b/charts/controller/files/reserved-name-patterns.txt
@@ -0,0 +1,12 @@
+^backup$
+^catalog$
+^cert-manager$
+^default$
+^drycc(?:-[\w-]+)?$
+^istio(?:-[\w-]+)?$
+^kube(?:-[\w-]+)?$
+^longhorn-system$
+^metallb$
+^mount-s3$
+^topolvm$
+^rook-ceph$
\ No newline at end of file
diff --git a/charts/controller/files/volume-claim-template.json b/charts/controller/files/volume-claim-template.json
new file mode 100644
index 000000000..be8133bfa
--- /dev/null
+++ b/charts/controller/files/volume-claim-template.json
@@ -0,0 +1,61 @@
+{% if type == 'csi' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "usage"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "{{storage_class}}",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeMode": "Filesystem"
+  }
+}
+{% elif type == 'nfs' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% elif type == 'oss' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% endif %}
diff --git a/charts/controller/files/volume-template.json b/charts/controller/files/volume-template.json
new file mode 100644
index 000000000..b4a54f132
--- /dev/null
+++ b/charts/controller/files/volume-template.json
@@ -0,0 +1,55 @@
+{% if type == 'nfs' %}
+{
+  "spec": {
+    "capacity": {
+      "storage": "65535Gi"
+    },
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "claimRef": {
+      "namespace": "{{namespace}}",
+      "name": "{{volume_claim_name}}"
+    },
+    "nfs": {
+      "server": "{{server}}",
+      "path": "{{path}}"
+    },
+    "mountOptions": [
+      "nfsvers=4.2"
+    ]
+  }
+}
+{% elif type == 'oss' %}
+{
+  "spec": {
+    "capacity": {
+      "storage": "65535Gi"
+    },
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "claimRef": {
+      "namespace": "{{namespace}}",
+      "name": "{{volume_claim_name}}"
+    },
+    "mountOptions": [
+      "allow-delete",
+      {% if path_style %}
+      "force-path-style",
+      {% endif %}
+      "aws-access-key-id {{access_key}}",
+      "aws-secret-access-key {{secret_key}}",
+      "endpoint-url {{server}}"
+    ],
+    "csi": {
+      "driver": "s3.csi.aws.com",
+      "volumeHandle": "{{volume_handle}}",
+      "volumeAttributes": {
+        "bucketName": "{{bucket}}"
+      }
+    }
+  }
+}
+{% endif %}
diff --git a/charts/controller/files/volume-usage-template.promql b/charts/controller/files/volume-usage-template.promql
new file mode 100644
index 000000000..fec7f1913
--- /dev/null
+++ b/charts/controller/files/volume-usage-template.promql
@@ -0,0 +1,12 @@
+max by(namespace, persistentvolumeclaim, storageclass) (
+  kube_persistentvolumeclaim_resource_requests_storage_bytes{
+    namespace=~"{{namespaces}}"
+  }
+  * on(namespace, persistentvolumeclaim) group_left(storageclass)
+  kube_persistentvolumeclaim_info{namespace=~"{{namespaces}}"}
+  * on(namespace, persistentvolumeclaim) group_left()
+  kube_persistentvolumeclaim_annotations{
+    namespace=~"{{namespaces}}",
+    annotation_billing_drycc_cc_type="usage"
+  }
+)
diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index a9cb0ea73..80902a4d2 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -197,291 +197,3 @@ env:
 - name: DRYCC_CONTROLLER_RUNNER
   value: mutate
 {{- end }}
-
-{{/* Generate controller config default limit specs */}}
-{{ define "controller.config.defaultLimitSpecs" }}
-- model: api.limitspec
-  pk: std1
-  fields:
-    cpu:
-      name: Universal CPU
-      cores: 32
-      clock: 3100MHZ
-      boost: 3700MHZ
-      threads: 64
-    memory:
-      size: 64GB
-      type: DDR4-ECC
-    features:
-      gpu:
-        name: Integrated GPU
-        tmus: 1
-        rops: 1
-        cores: 128
-        memory:
-          size: shared
-          type: shared
-      network: 10G
-    keywords:
-    - unknown
-    disabled: false
-    created: {{ now | date "2006-01-02T15:04:05.000Z" }}
-    updated: {{ now | date "2006-01-02T15:04:05.000Z" }}
-{{- end }}
-
-{{/* Generate controller config default limit plans */}}
-{{ define "controller.config.defaultLimitPlans" }}
-{{- $index := 0 }}
-{{- $cpus := tuple 1 2 4 8 16 32 64 }}
-{{- $scales := tuple 1 2 4 8 }}
-{{- range $cpu := $cpus }}
-{{- range $scale := $scales }}
-{{- $memory := mul $cpu $scale }}
-- model: api.limitplan
-  pk: std1.large.c{{ $cpu }}m{{ $memory }}
-  fields:
-    spec_id: std1
-    cpu: {{ $cpu }}
-    memory: {{ $memory }}
-    features:
-      gpu: 1
-      network: 1
-    disabled: false
-    priority: {{ add 10000 (mul (add1 $index) 100) }}
-    limits:
-      cpu: {{ $cpu }}
-      memory: {{ $memory }}Gi
-      ephemeral-storage: 4Gi
-    requests:
-      cpu: {{ divf $cpu 4 }}
-      memory: {{ divf $memory 2 }}Gi
-      ephemeral-storage: 2Gi
-    annotations:
-      kubernetes.io/egress-bandwidth: 100M
-      kubernetes.io/ingress-bandwidth: 100M
-    node_selector: {}
-    runtime_class_name: ""
-    pod_security_context: {}
-    container_security_context: {}
-    created: {{ now | date "2006-01-02T15:04:05.000Z" }}
-    updated: {{ now | date "2006-01-02T15:04:05.000Z" }}
-{{- $index = (add1 $index) }}
-{{- end }}
-{{- end }}
-{{- end }}
-
-{{/* Generate controller config default metrics */}}
-{{ define "controller.config.defaultMetrics" }}
-container_cpu_load_average_10s: [instance, namespace, pod, container]
-container_cpu_system_seconds_total: [instance, namespace, pod, container]
-container_cpu_usage_seconds_total: [instance, namespace, pod, container]
-container_cpu_user_seconds_total: [instance, namespace, pod, container]
-container_cpu_cfs_periods_total: [instance, namespace, pod, container]
-container_cpu_cfs_throttled_periods_total: [instance, namespace, pod, container]
-container_cpu_cfs_throttled_seconds_total: [instance, namespace, pod, container]
-container_fs_inodes_free: [instance, namespace, pod, container]
-container_fs_usage_bytes: [instance, namespace, pod, container]
-container_fs_inodes_total: [instance, namespace, pod, container]
-container_fs_io_current: [instance, namespace, pod, container]
-container_fs_io_time_seconds_total: [instance, namespace, pod, container]
-container_fs_io_time_weighted_seconds_total: [instance, namespace, pod, container]
-container_fs_limit_bytes: [instance, namespace, pod, container]
-container_fs_reads_bytes_total: [instance, namespace, pod, container]
-container_fs_read_seconds_total: [instance, namespace, pod, container]
-container_fs_reads_merged_total: [instance, namespace, pod, container]
-container_fs_reads_total: [instance, namespace, pod, container]
-container_fs_sector_reads_total: [instance, namespace, pod, container]
-container_fs_sector_writes_total: [instance, namespace, pod, container]
-container_fs_writes_bytes_total: [instance, namespace, pod, container]
-container_fs_write_seconds_total: [instance, namespace, pod, container]
-container_fs_writes_merged_total: [instance, namespace, pod, container]
-container_fs_writes_total: [instance, namespace, pod, container]
-container_blkio_device_usage_total: [instance, namespace, pod, container]
-container_memory_failures_total: [instance, namespace, pod, container]
-container_memory_failcnt: [instance, namespace, pod, container]
-container_memory_cache: [instance, namespace, pod, container]
-container_memory_mapped_file: [instance, namespace, pod, container]
-container_memory_max_usage_bytes: [instance, namespace, pod, container]
-container_memory_rss: [instance, namespace, pod, container]
-container_memory_swap: [instance, namespace, pod, container]
-container_memory_usage_bytes: [instance, namespace, pod, container]
-container_memory_working_set_bytes: [instance, namespace, pod, container]
-container_network_receive_bytes_total: [instance, namespace, pod, container]
-container_network_receive_errors_total: [instance, namespace, pod, container]
-container_network_receive_packets_dropped_total: [instance, namespace, pod, container]
-container_network_receive_packets_total: [instance, namespace, pod, container]
-container_network_transmit_bytes_total: [instance, namespace, pod, container]
-container_network_transmit_errors_total: [instance, namespace, pod, container]
-container_network_transmit_packets_dropped_total: [instance, namespace, pod, container]
-container_network_transmit_packets_total: [instance, namespace, pod, container]
-container_oom_events_total: [instance, namespace, pod, container]
-container_processes: [instance, namespace, pod, container]
-container_sockets: [instance, namespace, pod, container]
-container_file_descriptors: [instance, namespace, pod, container]
-container_threads: [instance, namespace, pod, container]
-container_threads_max: [instance, namespace, pod, container]
-container_ulimits_soft: [instance, namespace, pod, container]
-container_spec_cpu_quota: [instance, namespace, pod, container]
-container_spec_cpu_period: [instance, namespace, pod, container]
-container_spec_cpu_shares: [instance, namespace, pod, container]
-container_spec_memory_limit_bytes: [instance, namespace, pod, container]
-container_spec_memory_reservation_limit_bytes: [instance, namespace, pod, container]
-container_spec_memory_swap_limit_bytes: [instance, namespace, pod, container]
-container_start_time_seconds: [instance, namespace, pod, container]
-container_tasks_state: [instance, namespace, pod, container]
-container_last_seen: [instance, namespace, pod, container]
-container_accelerator_memory_used_bytes: [instance, namespace, pod, container]
-container_accelerator_memory_total_bytes: [instance, namespace, pod, container]
-container_accelerator_duty_cycle: [instance, namespace, pod, container]
-kube_pod_ips: [ip, ip_family, namespace, node, pod]
-kube_pod_container_status_running: [container, namespace, node, pod]
-kube_pod_container_status_ready: [container, namespace, node, pod]
-kube_pod_container_status_terminated: [container, namespace, node, pod]
-kube_pod_container_status_waiting: [container, namespace, node, pod]
-kube_pod_container_status_restarts_total: [container, namespace, node, pod]
-kube_pod_spec_volumes_persistentvolumeclaims_info: [namespace, pod, persistentvolumeclaim]
-kubelet_volume_stats_used_bytes: [namespace, persistentvolumeclaim]
-kubelet_volume_stats_available_bytes: [namespace, persistentvolumeclaim]
-kubelet_volume_stats_capacity_bytes: [namespace, persistentvolumeclaim]
-kubelet_volume_stats_inodes: [namespace, persistentvolumeclaim]
-kubelet_volume_stats_inodes_free: [namespace, persistentvolumeclaim]
-kubelet_volume_stats_inodes_used: [namespace, persistentvolumeclaim]
-istio_requests_total: [source_workload_namespace, reporter, destination_service, response_code, connection_security_policy, source_workload]
-istio_request_duration_milliseconds_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
-istio_request_bytes_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
-istio_response_bytes_bucket: [source_workload_namespace, reporter, destination_service, connection_security_policy, source_workload, le]
-{{- end }}
-
-{{/* Generate controller config default reserved name patterns */}}
-{{ define "controller.config.defaultReservedNamePatterns" }}
-^backup$
-^catalog$
-^cert-manager$
-^default$
-^drycc(?:-[\w-]+)?$
-^istio(?:-[\w-]+)?$
-^kube(?:-[\w-]+)?$
-^longhorn-system$
-^metallb$
-^mount-s3$
-^topolvm$
-^rook-ceph$
-{{- end }}
-
-{{/* Generate controller config default secrets template */}}
-{{ define "controller.config.defaultSecretTemplate" }}
-oss:
-  data:
-    key_id: $access_key
-    access_key: $secret_key
-{{- end }}
-
-{{/* Generate controller config default volume template */}}
-{{ define "controller.config.defaultVolumeTemplate" }}
-nfs:
-  spec:
-    capacity:
-      storage: 65535Gi
-    accessModes:
-    - ReadWriteMany
-    claimRef:
-      namespace: $namespace
-      name: $volume_claim_name
-    nfs:
-      server: $server
-      path: $path
-    mountOptions:
-    - nfsvers=4.2
-oss:
-  spec:
-    capacity:
-      storage: 65535Gi
-    accessModes:
-    - ReadWriteMany
-    storageClassName: ''
-    claimRef:
-      namespace: $namespace
-      name: $volume_claim_name
-    mountOptions:
-    - allow-delete
-    - force-path-style
-    - endpoint-url $server
-    csi:
-      driver: s3.csi.aws.com
-      volumeHandle: $volume_handle
-      volumeAttributes:
-        bucketName: $bucket
-        authenticationSource: secrets
-      nodePublishSecretRef:
-        name: $secret_name
-        namespace: $namespace
-{{- end }}
-
-{{/* Generate controller config default volume claim template */}}
-{{ define "controller.config.defaultVolumeClaimTemplate" }}
-csi:
-  metadata:
-    annotations:
-      billing.drycc.cc/type: usage
-  spec:
-    accessModes:
-    - ReadWriteMany
-    storageClassName: $storage_class
-    resources:
-      requests:
-        storage: $size
-    volumeMode: Filesystem
-nfs:
-  metadata:
-    annotations:
-      billing.drycc.cc/type: basic
-  spec:
-    accessModes:
-    - ReadWriteMany
-    storageClassName: ''
-    resources:
-      requests:
-        storage: $size
-    volumeName: $volume_name
-oss:
-  metadata:
-    annotations:
-      billing.drycc.cc/type: basic
-  spec:
-    accessModes:
-    - ReadWriteMany
-    storageClassName: ''
-    resources:
-      requests:
-        storage: $size
-    volumeName: $volume_name
-{{- end }}
-
-{{/* Generate controller config default volume usage template */}}
-{{ define "controller.config.defaultVolumeUsageTemplate" }}
-max by(namespace, persistentvolumeclaim, storageclass) (
-  kube_persistentvolumeclaim_resource_requests_storage_bytes{
-    namespace=~"${namespaces}"
-  }
-  * on(namespace, persistentvolumeclaim) group_left(storageclass)
-  kube_persistentvolumeclaim_info{namespace=~"${namespaces}"}
-  * on(namespace, persistentvolumeclaim) group_left()
-  kube_persistentvolumeclaim_annotations{
-    namespace=~"${namespaces}",
-    annotation_billing_drycc_cc_type="usage"
-  }
-)
-{{- end }}
-
-{{/* Generate controller config default network usage template */}}
-{{ define "controller.config.defaultNetworkUsageTemplate" }}
-label_replace(
-  sum(increase(istio_request_bytes_sum{namespace=~"${namespaces}"}[${duration}])) by (namespace,pod),
-  "direction", "rx", "", ""
-)
-or
-label_replace(
-  sum(increase(istio_response_bytes_sum{namespace=~"${namespaces}"}[${duration}])) by (namespace,pod),
-  "direction", "tx", "", ""
-)
-{{- end }}
diff --git a/charts/controller/templates/controller-configmap.yaml b/charts/controller/templates/controller-configmap.yaml
index d164f2105..18493da9a 100644
--- a/charts/controller/templates/controller-configmap.yaml
+++ b/charts/controller/templates/controller-configmap.yaml
@@ -9,53 +9,47 @@ data:
     {{- if .Values.config.metrics }}
     {{- (tpl .Values.config.metrics $) | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultMetrics" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{- .Files.Get "files/metrics.json" | nindent 4 }}
     {{- end }}
   limit-specs.json: |
     {{- if .Values.config.limitSpecs }}
     {{- (tpl .Values.config.limitSpecs $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultLimitSpecs" . | fromYamlArray | toPrettyJson | nindent 4 }}
+    {{- .Files.Get "files/limit-specs.json" | nindent 4 }}
     {{- end }}
   limit-plans.json: |
     {{- if .Values.config.limitPlans }}
     {{- (tpl .Values.config.limitPlans $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultLimitPlans" . | fromYamlArray | toPrettyJson | nindent 4 }}
-    {{- end }}
-  secret-template.json: |
-    {{- if .Values.config.secretTemplate }}
-    {{- (tpl .Values.config.secretTemplate $)  | nindent 4 }}
-    {{- else}}
-    {{- include "controller.config.defaultSecretTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{- .Files.Get "files/limit-plans.json" | nindent 4 }}
     {{- end }}
   volume-template.json: |
     {{- if .Values.config.volumeTemplate }}
     {{- (tpl .Values.config.volumeTemplate $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultVolumeTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{- .Files.Get "files/volume-template.json" | nindent 4 }}
     {{- end }}
   reserved-name-patterns.txt: |-
     {{- if .Values.config.reservedNames }}
     {{- (tpl .Values.config.reservedNames $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultReservedNamePatterns" . | nindent 4 }}
+    {{- .Files.Get "files/reserved-name-patterns.txt" | nindent 4 }}
     {{- end }}
   volume-claim-template.json: |
     {{- if .Values.config.volumeClaimTemplate }}
     {{- (tpl .Values.config.volumeClaimTemplate $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultVolumeClaimTemplate" . | fromYaml | toPrettyJson | nindent 4 }}
+    {{-  .Files.Get "files/volume-claim-template.json" | nindent 4 }}
     {{- end }}
   volume-usage-template.promql: |
     {{- if .Values.config.volumeUsageTemplate }}
     {{- (tpl .Values.config.volumeUsageTemplate $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultVolumeUsageTemplate" . | nindent 4 }}
+    {{- .Files.Get "files/volume-usage-template.promql" | nindent 4 }}
     {{- end }}
   network-usage-template.promql: |
     {{- if .Values.config.networkUsageTemplate }}
     {{- (tpl .Values.config.networkUsageTemplate $)  | nindent 4 }}
     {{- else}}
-    {{- include "controller.config.defaultNetworkUsageTemplate" . | nindent 4 }}
+    {{- .Files.Get "files/network-usage-template.promql" | nindent 4 }}
     {{- end }}
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 0068eec81..09d45813c 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -1,11 +1,11 @@
 import logging
-import string
 import copy
 import json
 import uuid
 import hashlib
 from django.db import models, transaction
 from django.conf import settings
+from django.template import Template, Context
 from django.contrib.auth import get_user_model
 from api.tasks import send_app_log
 from api.utils import unit_to_bytes, validate_label
@@ -44,14 +44,8 @@ def pv_name(self):
             hexdigest[20:]
         ]))
 
-    @property
-    def secret_name(self):
-        return self.pv_name
-
     @transaction.atomic
     def save(self, *args, **kwargs):
-        if self.type not in settings.DRYCC_VOLUME_CLAIM_TEMPLATE:
-            raise DryccException(f'Volume type {self.type} is not supported.')
         # Attach volume, updates k8s
         self.save_to_k8s()
         # check path
@@ -112,15 +106,11 @@ def check_path(self, path=None):
             raise DryccException(msg)
 
     def save_to_k8s(self):
-        if self.type in settings.DRYCC_SECRET_TEMPLATE:
-            self._save_secret()
         if self.type in settings.DRYCC_VOLUME_TEMPLATE:
             self._save_pv()
         self._save_pvc()
 
     def delete_from_k8s(self):
-        if self.type in settings.DRYCC_SECRET_TEMPLATE:
-            self._delete_secret()
         if self.type in settings.DRYCC_VOLUME_TEMPLATE:
             self._delete_pv()
         self._delete_pvc()
@@ -138,13 +128,15 @@ def _format_size(size):
     def _save_pv(self):
         kwds = copy.deepcopy(self.parameters.get(self.type, {}))
         kwds.update({
+            "type": self.type,
             "volume_claim_name": self.name,
             "namespace": self.app.id,
-            "secret_name": self.secret_name,
             "volume_handle": "%s" % uuid.uuid4(),
         })
-        t = string.Template(json.dumps(settings.DRYCC_VOLUME_TEMPLATE.get(self.type)))
-        kwargs = json.loads(t.safe_substitute(**kwds))
+        json_str = Template(settings.DRYCC_VOLUME_TEMPLATE).render(Context(kwds)).strip()
+        if not json_str:
+            return
+        kwargs = json.loads(json_str)
         try:
             self.scheduler.pv.get(self.pv_name)
             msg = "Volume {} already exists".format(self.pv_name)
@@ -170,14 +162,17 @@ def _delete_pv(self):
             logger.exception(e)
 
     def _save_pvc(self):
-        kwds = copy.deepcopy(self.parameters)
+        kwds = copy.deepcopy(self.parameters.get(self.type, {}))
         kwds.update({
+            "type": self.type,
             "size": self._format_size(self.size),
             "volume_name": self.pv_name,
             "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
         })
-        t = string.Template(json.dumps(settings.DRYCC_VOLUME_CLAIM_TEMPLATE.get(self.type)))
-        kwargs = json.loads(t.safe_substitute(**kwds))
+        json_str = Template(settings.DRYCC_VOLUME_CLAIM_TEMPLATE).render(Context(kwds)).strip()
+        if not json_str:
+            raise DryccException('Volume claim template render error, context: {}'.format(kwds))
+        kwargs = json.loads(json_str)
         try:
             self.scheduler.pvc.get(self.app.id, self.name)
             msg = "Volume claim {} already exists in this namespace".format(self.name)
@@ -200,32 +195,6 @@ def _delete_pvc(self):
         except KubeException as e:
             logger.exception(e)
 
-    def _save_secret(self):
-        kwds = copy.deepcopy(self.parameters.get(self.type, {}))
-        t = string.Template(json.dumps(settings.DRYCC_SECRET_TEMPLATE.get(self.type)))
-        kwargs = json.loads(t.safe_substitute(**kwds))
-        try:
-            self.scheduler.secret.get(self.app.id, self.secret_name)
-            msg = "Secret {} already exists".format(self.secret_name)
-            self.log(msg, logging.INFO)
-            self.scheduler.secret.patch(self.app.id, self.secret_name, **kwargs)
-        except KubeException as e:
-            logger.info(e)
-            try:
-                self.scheduler.secret.create(self.app.id, self.secret_name, **kwargs)
-            except KubeException as e:
-                msg = 'There was a problem creating the volume ' \
-                      '{} for {}'.format(self.secret_name, self.app_id)
-                raise ServiceUnavailable(msg) from e
-
-    def _delete_secret(self):
-        try:
-            # We raise an exception when a volume doesn't exist
-            self.scheduler.secret.get(self.app.id, self.secret_name)
-            self.scheduler.secret.delete(self.app.id, self.secret_name)
-        except KubeException as e:
-            logger.exception(e)
-
     class Meta:
         get_latest_by = 'created'
         unique_together = (('app', 'name'),)
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 2d02ca6f3..df8a1f79e 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -1,12 +1,12 @@
 import time
 import aiohttp
-from string import Template
 from typing import Iterator, AsyncGenerator
 from django.conf import settings
+from django.template import Template, Context
 
 
 query_last_metrics_promql_tpl = Template("""
-last_over_time({__name__=~"${metrics}",namespace="${namespace}"}[${duration}])
+last_over_time({__name__=~"{{metrics}}",namespace="{{namespace}}"}[{{duration}}])
 """)
 
 
@@ -25,11 +25,11 @@ async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
     if not settings.DRYCC_METRICS_CONFIG:
         return
     url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = query_last_metrics_promql_tpl.substitute(
-        metrics='|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
-        namespace=namespace,
-        duration=settings.DRYCC_METRICS_INTERVAL
-    )
+    promql = query_last_metrics_promql_tpl.render(Context({
+        "metrics": '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
+        "namespace": namespace,
+        "duration": settings.DRYCC_METRICS_INTERVAL
+    }))
     for item in await query_prom(url, {"query": promql, "start": int(time.time() - 60)}):
         yield '%s{%s} %s\n' % (
             item['metric']['__name__'],
@@ -46,17 +46,17 @@ async def query_volume_usage(namespaces: Iterator[str], start: int, stop: int
     if not settings.DRYCC_VICTORIAMETRICS_URL:
         return []
     url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = Template(settings.DRYCC_VOLUME_USAGE_TEMPLATE).substitute(
-        namespaces="|".join(namespaces)
-    )
+    promql = Template(settings.DRYCC_VOLUME_USAGE_TEMPLATE).render(Context({
+        "namespaces": "|".join(namespaces)
+    }))
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
 
 
 async def query_network_usage(namespaces: Iterator[str], start: int, stop: int
                               ) -> list[tuple[dict[str, str], int]]:
     url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
-    promql = Template(settings.DRYCC_NETWORK_USAGE_TEMPLATE).substitute(
-        namespaces="|".join(namespaces),
-        duration=f"{stop-start}s"
-    )
+    promql = Template(settings.DRYCC_NETWORK_USAGE_TEMPLATE).render(Context({
+        "namespaces": "|".join(namespaces),
+        "duration": f"{stop-start}s"
+    }))
     return await query_prom(url, {"query": promql, "start": start, "end": stop})
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 42e1c4721..9b4dafc86 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -286,21 +286,13 @@ def randstr(k):
 DRYCC_METRICS_EXPIRY = int(os.environ.get('DRYCC_METRICS_EXPIRY', '20'))
 DRYCC_METRICS_INTERVAL = os.environ.get('DRYCC_METRICS_INTERVAL', '5m')
 
-# drycc secret template
-DRYCC_SECRET_TEMPLATE = {}
-DRYCC_SECRET_TEMPLATE_PATH = os.environ.get(
-    'DRYCC_SECRET_TEMPLATE_PATH', '/etc/controller/secret-template.json')
-if os.path.exists(DRYCC_SECRET_TEMPLATE_PATH):
-    with open(DRYCC_SECRET_TEMPLATE_PATH) as fd:
-        DRYCC_SECRET_TEMPLATE = json.load(fd)
-
 # drycc volume template
 DRYCC_VOLUME_TEMPLATE = {}
 DRYCC_VOLUME_TEMPLATE_PATH = os.environ.get(
     'DRYCC_VOLUME_TEMPLATE_PATH', '/etc/controller/volume-template.json')
 if os.path.exists(DRYCC_VOLUME_TEMPLATE_PATH):
     with open(DRYCC_VOLUME_TEMPLATE_PATH) as fd:
-        DRYCC_VOLUME_TEMPLATE = json.load(fd)
+        DRYCC_VOLUME_TEMPLATE = fd.read()
 
 # drycc volume claim template
 DRYCC_VOLUME_CLAIM_TEMPLATE = {}
@@ -308,7 +300,7 @@ def randstr(k):
     'DRYCC_VOLUME_CLAIM_TEMPLATE_PATH', '/etc/controller/volume-claim-template.json')
 if os.path.exists(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH):
     with open(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH) as fd:
-        DRYCC_VOLUME_CLAIM_TEMPLATE = json.load(fd)
+        DRYCC_VOLUME_CLAIM_TEMPLATE = fd.read()
 
 # drycc volume usage template
 DRYCC_VOLUME_USAGE_TEMPLATE = ""
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index 2ed583b93..2f2352354 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -61,33 +61,66 @@ def __getitem__(self, item):
 WORKFLOW_MANAGER_ACCESS_KEY = "1234567890"
 WORKFLOW_MANAGER_SECRET_KEY = "1234567890"
 
-DRYCC_VOLUME_CLAIM_TEMPLATE = {
-    "csi": {
-        "spec": {
-            "accessModes": [
-                "ReadWriteMany"
-            ],
-            "resources": {
-                "requests": {
-                    "storage": "$size"
-                }
-            },
-            "storageClassName": "$storage_class",
-            "volumeMode": "Filesystem"
-        }
+DRYCC_VOLUME_CLAIM_TEMPLATE = """
+{% if type == 'csi' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "usage"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "{{storage_class}}",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeMode": "Filesystem"
+  }
+}
+{% elif type == 'nfs' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
     },
-    "nfs": {
-        "spec": {
-            "accessModes": [
-                "ReadWriteMany"
-            ],
-            "resources": {
-                "requests": {
-                    "storage": "$size"
-                }
-            },
-            "storageClassName": "",
-            "volumeName": "$volume_name"
-        }
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% elif type == 'nfs' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
     }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
 }
+{% endif %}
+"""
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index bea1e4656..0ce9b9f7e 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -206,7 +206,8 @@ def test_volume_mount(self, mock_requests):
             {'name': 'myvolume1', 'size': '500G'}
         ]
         for _ in data:
-            self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+            response = self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+            self.assertEqual(response.status_code, 201, response.data)
 
         self.build_deploy(app_id)
         # mount

From 38585697ebf04d11663127b9763d58eb713774c2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 6 Nov 2025 15:32:05 +0800
Subject: [PATCH 184/200] chore(deps): bump django in /rootfs in the pip group
 across 1 directory (#199)

Bumps the pip group with 1 update in the /rootfs directory: [django](https://github.com/django/django).


Updates `django` from 5.2.7 to 5.2.8
- [Commits](https://github.com/django/django/compare/5.2.7...5.2.8)

---
updated-dependencies:
- dependency-name: django
  dependency-version: 5.2.8
  dependency-type: direct:production
  dependency-group: pip
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 22e689410..4896da53d 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.10
 backoff==2.2.1
-django==5.2.7
+django==5.2.8
 channels==4.3.1
 aiohttp==3.12.15
 django-cors-headers==4.8.0

From bc6567a4501e27a4bef965b554f35f6bb9753102 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 6 Nov 2025 15:32:35 +0800
Subject: [PATCH 185/200] chore(controller): change env to build arg

---
 rootfs/Dockerfile      |  2 +-
 rootfs/Dockerfile.test | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile
index 4f151964c..08f17c695 100644
--- a/rootfs/Dockerfile
+++ b/rootfs/Dockerfile
@@ -1,7 +1,7 @@
 ARG CODENAME
 FROM registry.drycc.cc/drycc/base:${CODENAME}
 
-ENV DRYCC_UID=1001 \
+ARG DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/workspace \
   PYTHON_VERSION="3.13"
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index f81dded4d..93ec2b24d 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -1,17 +1,17 @@
 ARG CODENAME
 FROM registry.drycc.cc/drycc/base:${CODENAME}
 
-ARG DRYCC_HOME_DIR=/workspace
-
-COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt
-COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
-
-ENV PGDATA="/opt/drycc/postgresql/data" \
+ARG DRYCC_HOME_DIR=/workspace \
   PYTHON_VERSION="3.13" \
   VALKEY_VERSION="8.1.3" \
   POSTGRES_VERSION="17.6" \
   GOSU_VERSION="1.18"
 
+COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt
+COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
+
+ENV PGDATA="/opt/drycc/postgresql/data"
+
 RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
   install-packages mercurial ca-certificates git inotify-tools $buildDeps \
   && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \

From ee60f02c2b65de96fdaac01dad370f5cebe85770 Mon Sep 17 00:00:00 2001
From: lijianguo <lijianguo1991@outlook.com>
Date: Thu, 13 Nov 2025 11:58:04 +0800
Subject: [PATCH 186/200] chore(controller): process do not mount a
 serviceaccount (#200)

---
 rootfs/scheduler/resources/deployment.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index 0a8b2bae9..59b0dca8e 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -107,6 +107,7 @@ def manifest(self, namespace, name, image, command, args, spec_annotations, **kw
 
         # set the old deployment spec annotations on this deployment
         manifest['spec']['template']['metadata']['annotations'] = spec_annotations
+        manifest['spec']['template']['spec']['automountServiceAccountToken'] = False
         if annotations:
             manifest['spec']['template']['metadata']['annotations'].update(annotations)
         if "resource_version" in kwargs:

From 613b323633aeef45986311917f75c986c39f9236 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Sat, 15 Nov 2025 21:19:02 +0800
Subject: [PATCH 187/200] chore(python): bump version to 3.14

---
 rootfs/Dockerfile      | 2 +-
 rootfs/Dockerfile.test | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile
index 08f17c695..ae5a3b607 100644
--- a/rootfs/Dockerfile
+++ b/rootfs/Dockerfile
@@ -4,7 +4,7 @@ FROM registry.drycc.cc/drycc/base:${CODENAME}
 ARG DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/workspace \
-  PYTHON_VERSION="3.13"
+  PYTHON_VERSION="3.14"
 
 RUN groupadd drycc --gid ${DRYCC_GID} \
   && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR}
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index 93ec2b24d..74f4c8c91 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -2,10 +2,10 @@ ARG CODENAME
 FROM registry.drycc.cc/drycc/base:${CODENAME}
 
 ARG DRYCC_HOME_DIR=/workspace \
-  PYTHON_VERSION="3.13" \
-  VALKEY_VERSION="8.1.3" \
-  POSTGRES_VERSION="17.6" \
-  GOSU_VERSION="1.18"
+  PYTHON_VERSION="3.14" \
+  VALKEY_VERSION="9.0.0" \
+  POSTGRES_VERSION="18.1" \
+  GOSU_VERSION="1.19"
 
 COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt
 COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt

From 827505c1a15eb11cc663f6b3bdcf4f47645c2f2a Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Mon, 17 Nov 2025 10:09:20 +0800
Subject: [PATCH 188/200] fix(test): apps aren't loaded yet.

---
 rootfs/api/settings/testing.py |  5 +++++
 rootfs/requirements.txt        | 22 +++++++++++-----------
 2 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index 2f2352354..e601ffff0 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -1,11 +1,16 @@
 import random
 import string
 import os
+import multiprocessing
 from django.core import signals
 from api.settings.celery import app
 from api.settings.production import DATABASES
 from api.settings.production import *  # noqa
 
+# Fix Django test error.
+# https://github.com/django/django/blob/main/django/test/runner.py#L455
+# This code was removed in Django > 5.2, so it is safe to remove this when we upgrade.
+multiprocessing.set_start_method('fork')
 # Monkey patch celery
 app.conf.update(task_always_eager=True)
 signals.request_started.send = lambda sender, **named: []
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 4896da53d..b39a97da9 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,29 +1,29 @@
 # Drycc controller requirements
-inotify==0.2.10
+inotify==0.2.12
 backoff==2.2.1
 django==5.2.8
 channels==4.3.1
-aiohttp==3.12.15
-django-cors-headers==4.8.0
-django-guardian==3.1.3
+aiohttp==3.13.2
+django-cors-headers==4.9.0
+django-guardian==3.2.0
 djangorestframework==3.16.1
 docker==7.1.0
 gunicorn==23.0.0
-uvicorn[standard]==0.35.0
-asgiref==3.9.1
-idna==3.10
+uvicorn[standard]==0.38.0
+asgiref==3.10.0
+idna==3.11
 jsonschema==4.25.1
 morph==0.1.5
 packaging==25.0
 pyasn1==0.6.1
-psycopg[binary]==3.2.10
-pyOpenSSL==25.1.0
+psycopg[binary]==3.2.12
+pyOpenSSL==25.3.0
 ndg-httpsclient==0.5.1
 pytz==2025.2
 requests==2.32.5
 requests-toolbelt==1.0.0
 celery[redis]==5.5.3
-hiredis==3.2.1
+hiredis==3.3.0
 dj-database-url==3.0.1
 social-auth-app-django==5.6.0
-kubernetes==33.1.0
+kubernetes==34.1.0

From e611c434a607f7b8670589f6c5f7bb46f23e660e Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 21 Nov 2025 23:05:38 +0800
Subject: [PATCH 189/200] chore(dryccfile): override all config when using
 dryccfile

---
 rootfs/api/models/build.py      | 31 +++++++++++--------------------
 rootfs/api/tests/test_config.py | 10 +++++-----
 2 files changed, 16 insertions(+), 25 deletions(-)

diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 41da3ddc2..cbd097523 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -120,32 +120,23 @@ def _get_or_create_config(self):
         """
         dryccfile to config
         """
-        config_values, config_values_ref, changed_fields, replace_groups = [], {}, set(), set()
+        if not self.dryccfile:
+            return self.app.release_set.filter(failed=False).latest().config
+        # create config from dryccfile
+        values, values_ref = [], {}
         for group, envs in self.dryccfile.get('config', {}).items():
             for key, value in envs.items():
-                replace_groups.add(group)
-                config_values.append({"name": key, "group": group, "value": value})
-            changed_fields.update(["values", "values_refs"])
-
-        replace_ptypes = set()
+                values.append({"name": key, "group": group, "value": value})
         for pipeline in self.dryccfile.get('pipeline', {}).values():
             if 'env' in pipeline or 'config' in pipeline:
                 for key, value in pipeline.get('env', {}).items():
-                    config_values.append({"name": key, "ptype": pipeline['ptype'], "value": value})
+                    values.append({"name": key, "ptype": pipeline['ptype'], "value": value})
                 for config_ref in pipeline.get('config', []):
-                    if pipeline['ptype'] not in config_values_ref:
-                        config_values_ref[pipeline['ptype']] = [config_ref]
+                    if pipeline['ptype'] not in values_ref:
+                        values_ref[pipeline['ptype']] = [config_ref]
                     else:
-                        config_values_ref[pipeline['ptype']].append(config_ref)
-                replace_ptypes.add(pipeline['ptype'])
-                changed_fields.update(["values", "values_refs"])
-
-        old_config = self.app.release_set.filter(failed=False).latest().config
-        if not changed_fields:
-            return old_config
+                        values_ref[pipeline['ptype']].append(config_ref)
         config = Config(
-            owner=self.owner, app=self.app, values=config_values, values_refs=config_values_ref)
-        config.merge_field("values", old_config, replace_ptypes, replace_groups)
-        config.merge_field("values_refs", old_config, replace_ptypes)
-        config.save(ignore_update_fields=changed_fields)
+            owner=self.owner, app=self.app, values=values, values_refs=values_ref)
+        config.save(ignore_update_fields=["values", "values_refs"])
         return config
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 3d310cf0d..b9d85c776 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -1001,8 +1001,8 @@ def test_config_from_dryccfile(self, mock_requests):
         self.assertEqual(
             release.config.envs("web"),
             {
-                'WEBSITE': 'www.drycc.cc', 'GROUP': 'g1', 'DEBUG': 'tr',
-                'PENV1': 'web', 'PENV2': 'web',  'TEST1': 'g1', 'TEST2': 'tr',
+                'GROUP': 'g1', 'DEBUG': 'tr', 'TEST1': 'g1',
+                'TEST2': 'tr', 'PENV1': 'web', 'PENV2': 'web'
             },
         )
 
@@ -1018,8 +1018,8 @@ def test_config_from_dryccfile(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.failed, False)
-        self.assertEqual(release.config.values, release.previous().config.values)
-        self.assertEqual(release.config.values_refs, release.previous().config.values_refs)
+        self.assertEqual(release.config.values, [])
+        self.assertEqual(release.config.values_refs, {})
         # set empty
         new_build_body = copy.deepcopy(build_body)
         new_build_body['dryccfile']['config'] = {}
@@ -1032,5 +1032,5 @@ def test_config_from_dryccfile(self, mock_requests):
             self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
         self.assertEqual(release.failed, False)
-        self.assertEqual(release.config.envs("web"), {'WEBSITE': 'www.drycc.cc'})
+        self.assertEqual(release.config.envs("web"), {})
         self.assertEqual(release.config.values_refs, {})

From 2835981bc264a3e2043e9d262b44ec810a6bebdd Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 28 Nov 2025 16:04:25 +0800
Subject: [PATCH 190/200] feat(consumers): add filer proxy

---
 rootfs/api/asgi.py                 |  24 +-
 rootfs/api/consumers.py            | 136 ++++-
 rootfs/api/filer.py                | 132 ++--
 rootfs/api/middleware.py           |  38 +-
 rootfs/api/routing.py              |  10 +-
 rootfs/api/settings/production.py  |  13 +-
 rootfs/api/settings/testing.py     |   6 +-
 rootfs/api/tests/test_consumers.py | 932 +++++++++++++++++++++++++++++
 rootfs/api/urls.py                 |   9 -
 rootfs/api/views.py                |  51 +-
 rootfs/requirements.txt            |   2 +-
 11 files changed, 1194 insertions(+), 159 deletions(-)
 create mode 100644 rootfs/api/tests/test_consumers.py

diff --git a/rootfs/api/asgi.py b/rootfs/api/asgi.py
index 64eb77edb..4f0063cf8 100644
--- a/rootfs/api/asgi.py
+++ b/rootfs/api/asgi.py
@@ -9,18 +9,24 @@
 
 import os
 
+from django.urls import re_path
 from django.core.asgi import get_asgi_application
 from channels.routing import ProtocolTypeRouter, URLRouter
 from channels.security.websocket import AllowedHostsOriginValidator
 
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.settings.production')
-http = get_asgi_application()
-
-from .routing import websocket_urlpatterns  # noqa
-from .middleware import ChannelOAuthMiddleware  # noqa
-websocket = AllowedHostsOriginValidator(ChannelOAuthMiddleware(URLRouter(
-    websocket_urlpatterns
-)))
-
-application = ProtocolTypeRouter({"http": http, "websocket": websocket})
+django_asgi_app = get_asgi_application()
+
+from .routing import http_urlpatterns, websocket_urlpatterns  # noqa
+from .middleware import ChannelOAuthMiddleware, ChannelAPIVersionMiddleware  # noqa
+
+application = ProtocolTypeRouter({
+    "http": ChannelAPIVersionMiddleware(URLRouter([
+        *http_urlpatterns,
+        re_path(r'', django_asgi_app),
+    ])),
+    "websocket": AllowedHostsOriginValidator(ChannelOAuthMiddleware(URLRouter(
+        websocket_urlpatterns
+    )))
+})
diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index 1ff7450f9..e3995d771 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -2,7 +2,9 @@
 import time
 import six
 import ssl
+import aiohttp
 import asyncio
+from urllib.parse import urljoin
 from django.conf import settings
 from django.core.cache import cache
 
@@ -14,38 +16,47 @@
 from kubernetes.stream import stream
 from kubernetes.stream.ws_client import STDOUT_CHANNEL, STDERR_CHANNEL, ERROR_CHANNEL
 
-from channels.db import database_sync_to_async
 from channels.exceptions import DenyConnection
+from channels.generic.http import AsyncHttpConsumer
 from channels.generic.websocket import AsyncWebsocketConsumer
 
 from .models.app import App
+from .models.volume import Volume
 from .permissions import has_app_permission
 
 
-class BaseAppConsumer(AsyncWebsocketConsumer):
+class AppPermChecker(object):
     timeout = 60 * 60
 
-    @database_sync_to_async
-    def has_perm(self):
+    def __init__(self, scope):
+        self.scope = scope
+
+    async def has_perm(self):
         if self.scope["user"] is None:
             return False, "user not login"
-        key = f"permission:user:{self.scope["user"].id}:app:{self.id}"
-        permission = cache.get(key)
+        app_id = self.scope["url_route"]["kwargs"]["id"]
+        key = f"permission:user:{self.scope["user"].id}:app:{app_id}"
+        permission = await cache.aget(key)
         if permission is None:
             try:
-                app = App.objects.get(id=self.id)
-                permission = has_app_permission(self.scope["user"], app, "GET")
+                app = await App.objects.aget(id=app_id)
+                permission = await sync_to_async(has_app_permission)(
+                    self.scope["user"], app, "GET")
                 if permission[0]:
-                    cache.set(key, permission, timeout=self.timeout)
+                    await cache.aset(key, permission, timeout=self.timeout)
             except App.DoesNotExist:
-                permission = (False, "user not exists")
+                permission = (False, "app not exists")
         return permission
 
+
+class BaseAppConsumer(AsyncWebsocketConsumer):
+
     async def connect(self):
-        self.id = self.scope["url_route"]["kwargs"]["id"]
-        is_ok, message = await self.has_perm()
+        app_perm_checker = AppPermChecker(self.scope)
+        is_ok, message = await app_perm_checker.has_perm()
         if is_ok:
             await self.accept()
+            self.id = self.scope["url_route"]["kwargs"]["id"]
         else:
             raise DenyConnection(message)
 
@@ -74,7 +85,7 @@ async def connect(self):
         self.conneted = True
         self.buffer = b''
         self.delimiter = b"\r\n"
-        self.pod_id = self.scope["url_route"]["kwargs"]["pod_id"]
+        self.pod_name = self.scope["url_route"]["kwargs"]["name"]
 
     def reader(self, sock):
         self.buffer += sock.read()
@@ -98,7 +109,7 @@ async def receive(self, text_data=None, bytes_data=None):
             return
         self.running = True
         data = json.loads(text_data)
-        args = (self.pod_id, self.id)
+        args = (self.pod_name, self.id)
         lines = data.get("lines", 300)
         follow = data.get("follow", False)
         previous = data.get("previous", False)
@@ -134,7 +145,7 @@ async def connect(self):
         await super().connect()
         self.stream = None
         self.conneted = True
-        self.pod_id = self.scope["url_route"]["kwargs"]["pod_id"]
+        self.pod_name = self.scope["url_route"]["kwargs"]["name"]
 
     async def send(self, data, channel=STDOUT_CHANNEL):
         channel_prefix = chr(channel)
@@ -177,7 +188,7 @@ async def disconnect(self, close_code):
 
     async def receive(self, text_data=None, bytes_data=None):
         if self.stream is None and text_data is not None:
-            args = (self.kubernetes.connect_get_namespaced_pod_exec, self.pod_id, self.id)
+            args = (self.kubernetes.connect_get_namespaced_pod_exec, self.pod_name, self.id)
             kwargs = json.loads(text_data)
             kwargs.update({"stderr": True, "stdout": True, "_preload_content": False})
             self.stream = await sync_to_async(stream)(*args, **kwargs)
@@ -188,3 +199,96 @@ async def receive(self, text_data=None, bytes_data=None):
             await sync_to_async(self.stream.write_channel)(channel, data)
         else:
             raise ValueError("This operation is not supported!")
+
+
+class FilerProxyConsumer(AsyncHttpConsumer):
+    from .middleware import ChannelOAuthMiddleware
+    chunk_size = 64 * 1024
+    middleware = ChannelOAuthMiddleware(None)
+    SKIP_REQUEST_HEADERS = {
+        'host', 'connection', 'keep-alive', 'proxy-connection', 'te', 'trailers', 'upgrade',
+    }
+    SKIP_RESPONSE_HEADERS = {
+        'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers',
+        'transfer-encoding', 'upgrade',
+    }
+
+    async def handle(self, body: bytes):
+        path = self.scope["url_route"]["kwargs"]["path"]
+        client = await self._get_client(url_path=f"{self.scope["path"].removesuffix(path)}webdav/")
+        if not client:
+            await self.send_response(status=404, body=b'app or volume not found')
+            return
+        if path in ['_/ping', '_/bind']:  # need authentication
+            await self._handle_controller_request(client, path)
+            return
+        elif not path.startswith('webdav/') or (filer := await client.info()) is None:
+            await self.send_response(status=428, body=b'filer service unavailable')
+            return
+        filer_target_url = "{}?{}".format(
+            urljoin(filer["endpoint"], self.scope["path"]), self.scope.get("query_string"))
+        method = self.scope['method'].upper()
+        headers = {
+            name_bytes.decode('latin-1').lower(): value_bytes.decode('latin-1')
+            for name_bytes, value_bytes in self.scope.get('headers', [])
+            if name_bytes.decode('latin-1').lower() not in self.SKIP_REQUEST_HEADERS
+        }
+        await self._handle_proxy_request(filer_target_url, method, headers, body)
+
+    async def _get_client(self, url_path):
+        try:
+            from .filer import FilerClient
+            app = await App.objects.aget(id=self.scope["url_route"]["kwargs"]["id"])
+            volume = await Volume.objects.filter(
+                app=app, name=self.scope["url_route"]["kwargs"]["name"]).afirst()
+            if not volume:
+                return None
+            return FilerClient(app.id, volume, url_path)
+        except App.DoesNotExist:
+            return None
+
+    async def _forward_response(self, response: aiohttp.ClientResponse):
+        response_headers = [
+            [name.encode('latin-1'), value.encode('latin-1')]
+            for name, value in response.headers.items()
+            if name.lower() not in self.SKIP_RESPONSE_HEADERS
+        ]
+        await self.send_headers(status=response.status, headers=response_headers)
+        async for chunk in response.content.iter_chunked(self.chunk_size):
+            if chunk:
+                await self.send_body(chunk, more_body=True)
+        # Send final empty chunk to indicate end of body
+        await self.send_body(b'', more_body=False)
+
+    async def _handle_proxy_request(self, url, method: str, headers: dict[str, str], data: bytes):
+        async with aiohttp.ClientSession(
+            timeout=aiohttp.ClientTimeout(total=settings.DRYCC_FILER_DURATION)
+        ) as session:
+            try:
+                async with session.request(
+                    method=method, url=url, headers=headers, data=data, allow_redirects=False
+                ) as response:
+                    await self._forward_response(response)
+            except aiohttp.ClientError as e:
+                await self.send_response(502, f"proxy service unavailable: {e}".encode('utf-8'))
+            except asyncio.TimeoutError:
+                await self.send_response(504, b'proxy request to backend filer timeout')
+
+    async def _handle_controller_request(self, client, path: str):
+        await self.middleware.login(self.scope)
+        app_perm_checker = AppPermChecker(self.scope)
+        is_ok, message = await app_perm_checker.has_perm()
+        status, body = 200, b''
+        if not is_ok:
+            status, body = 403, message.encode('utf-8')
+        elif path == '_/ping':
+            if (filer := await client.info()) is not None:
+                body = b'pong'
+            else:
+                status, body = 503, b'filer service unavailable'
+        elif path == '_/bind':
+            filer = await client.bind()
+            body = json.dumps({
+                "username": filer["username"], "password": filer["password"],
+            }).encode('utf-8')
+        await self.send_response(status, body)
diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
index f0d8ebf3c..5e2d092da 100644
--- a/rootfs/api/filer.py
+++ b/rootfs/api/filer.py
@@ -1,105 +1,111 @@
 import uuid
 import logging
-import requests
+import asyncio
+import aiohttp
 from django.conf import settings
 from django.core.cache import cache
-from requests.auth import HTTPBasicAuth
+from asgiref.sync import sync_to_async
 
+from .exceptions import DryccException
 from .tasks import send_app_log
-from .utils import random_string, get_httpclient, CacheLock
+from .utils import random_string, CacheLock, get_scheduler
 
 logger = logging.getLogger(__name__)
 
 
 class FilerClient(object):
 
-    def __init__(self, app_id, volume, scheduler):
-        self.bind = ":9000"
+    def __init__(self, app_id, volume, url_path):
         self.path = "/data"
+        self.ports = (9000, 9100)
         self.app_id = app_id
         self.volume = volume
-        self.scheduler = scheduler
+        self.url_path = url_path
+        self.scheduler = get_scheduler()
 
     def log(self, message, level=logging.INFO):
         logger.log(level, "[{}]: {}".format(self.app_id, message))
-        send_app_log.delay(self.app_id, message, level)
+        send_app_log.delay(self.app_id, message, logging.INFO)
 
-    @property
-    def server(self):
+    async def bind(self):
         lock_key = f"filer:lock:{self.app_id}:{self.volume.name}"
         lock = CacheLock(lock_key)
         try:
-            lock.acquire()
-            _server = cache.get(self.cache_key, None)
-            if not _server or not self.health(_server):
-                _server = self.get_server()
-                cache.set(self.cache_key, _server, timeout=settings.DRYCC_FILER_DURATION)
+            await sync_to_async(lock.acquire)()
+            if _filer := await self.info():
+                return _filer
+            else:
+                _filer = await sync_to_async(self._filer_bind)()
+                await cache.aset(self._cache_key, _filer, timeout=settings.DRYCC_FILER_DURATION)
         finally:
-            lock.release()
-        return _server
+            await sync_to_async(lock.release)()
+        return _filer
+
+    async def info(self):
+        _filer = await cache.aget(self._cache_key, None)
+        if _filer and await self._check_health(_filer):
+            await cache.atouch(self._cache_key, timeout=settings.DRYCC_FILER_DURATION)
+            return _filer
+        return None
 
     @property
-    def cache_key(self):
+    def _cache_key(self):
         return f"filer:{self.app_id}:{self.volume.name}"
 
-    def get_server(self):
-        self.clean()  # clean old filer
-        pod_name = f"drycc-filer-{uuid.uuid4().hex}"
-        k8s_volume = {"name": self.volume.name}
+    def _filer_bind(self):
+        username, password = random_string(32), random_string(32)
+        filer = {"username": username, "password": password}
+        job_ip = self._get_job_ip(self._create_job(username, password))
+        filer.update({
+            "endpoint": f"http://{job_ip}:{self.ports[1]}",
+            "ping_url": f"http://{job_ip}:{self.ports[0]}/_/ping",
+        })
+        return filer
+
+    def _create_job(self, username, password: str) -> str:
+        job_name, k8s_volume = f"drycc-filer-{uuid.uuid4()}", {"name": self.volume.name}
         if self.volume.type == "csi":
             k8s_volume.update({"persistentVolumeClaim": {"claimName": self.volume.name}})
         else:
             k8s_volume.update(self.volume.parameters)
-        username, password = random_string(32), random_string(32)
-        self.scheduler.pod.create(self.app_id, pod_name, settings.DRYCC_FILER_IMAGE, **{
+        self.scheduler.job.create(self.app_id, job_name, settings.DRYCC_FILER_IMAGE, **{
+            "command": ["init-stack", "/usr/bin/pingguard"],
             "args": [
-                "filer",
-                "--bind", self.bind, "--path", self.path,
-                "--duration", f"{settings.DRYCC_FILER_DURATION}",
-                "--waittime", f"{settings.DRYCC_FILER_WAITTIME}",
-                "--username", f"{username}", "--password", f"{password}",
+                f"--bind=:{self.ports[0]}", f"--interval={settings.DRYCC_FILER_DURATION}s",
+                "--",
+                "rclone", "serve", "webdav", self.path, f"--addr=:{self.ports[1]}",
+                f"--user={username}", f"--pass={password}", f"--baseurl={self.url_path}",
             ],
-            "labels": {"app": self.app_id, "pod": pod_name, "volume": self.volume.name},
             "app_type": "filer", "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
             "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
             "restart_policy": "Never", "image_pull_policy": settings.DRYCC_FILER_IMAGE_PULL_POLICY,
             "pod_security_context": {"fsGroup": 1001, "runAsGroup": 1001, "runAsUser": 1001},
+            "active_deadline_seconds": 2 ** 32,
+            "ttl_seconds_after_finished": settings.DRYCC_FILER_DURATION,
         })
-        address = self.scheduler.pod.get(self.app_id, pod_name).json()["status"]["podIP"]
-        return {"address": address, "username": username, "password": password}
+        return job_name
 
-    def clean(self):
-        response = self.scheduler.pod.get(
-            self.app_id, labels={"app": self.app_id, "type": "filer"})
-        if response.status_code != 200:
-            self.log("clean up old filter errors")
-            return False
-        for item in response.json()["items"]:
-            if item['status']['phase'] in ('Succeeded', 'Failed'):
-                pod_name = item['metadata']['name']
-                self.scheduler.pod.delete(self.app_id, pod_name)
-        self.log("clean up old filter completed")
-        return True
+    def _get_job_ip(self, job_name: str) -> str:
+        state, labels = 'initializing', {'job-name': job_name}
+        for count, state in enumerate(self.scheduler.pod.watch(
+            self.app_id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT,
+            until_states=['up', 'down', 'crashed', 'error'],
+        )):
+            self.log(f"waiting for filer bind: {state} * {count}")
+        if state != 'up':
+            raise DryccException(f'filer startup failed, current status: {state}')
+        pods = self.scheduler.pod.get(self.app_id, labels=labels).json()
+        if not pods["items"]:
+            raise DryccException('filer pod not found after run completed')
+        return pods["items"][0]["status"]["podIP"]
 
-    def health(self, server):
+    async def _check_health(self, filer):
         try:
-            return self.request(
-                "OPTIONS", server, timeout=2).headers.get('server') == 'drycc-filer'
-        except requests.exceptions.Timeout:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(
+                    filer["ping_url"],
+                    timeout=aiohttp.ClientTimeout(total=5),
+                ) as response:
+                    return response.status == 200
+        except (aiohttp.ClientError, asyncio.TimeoutError):
             return False
-
-    def request(self, method, server, path="/", **kwargs):
-        cache.touch(self.cache_key, timeout=settings.DRYCC_FILER_DURATION)
-        url = f"http://{server["address"]}:{self.bind.split(":")[1]}/{path}"
-        kwargs["auth"] = HTTPBasicAuth(server["username"], server["password"])
-        with get_httpclient() as session:
-            session.request(method, url, **kwargs)
-
-    def get(self, path, **kwargs):
-        return self.request("GET", self.server, path, **kwargs)
-
-    def post(self, path, **kwargs):
-        return self.request("POST", self.server, path, **kwargs)
-
-    def delete(self, path, **kwargs):
-        return self.request("DELETE", self.server, path, **kwargs)
diff --git a/rootfs/api/middleware.py b/rootfs/api/middleware.py
index e897443d5..47cc095be 100644
--- a/rootfs/api/middleware.py
+++ b/rootfs/api/middleware.py
@@ -3,15 +3,18 @@
 
 See https://docs.djangoproject.com/en/1.11/topics/http/middleware/
 """
-from api import __version__
+
 from django.http.request import HttpRequest
 
 from channels.middleware import BaseMiddleware
 from channels.db import database_sync_to_async
-from api.authentication import DryccAuthentication
+
+from api import __version__
+
+from .authentication import DryccAuthentication
 
 
-class APIVersionMiddleware(object):
+class DjangoAPIVersionMiddleware(object):
     """
     Include that REST API version with each response.
     """
@@ -32,6 +35,27 @@ def __call__(self, request):
         return response
 
 
+class ChannelAPIVersionMiddleware(BaseMiddleware):
+    """
+    Channel middleware to include REST API version with each HTTP response.
+    """
+    async def __call__(self, scope, receive, send):
+        if scope["type"] == "http":
+            async def send_with_version(message):
+                if message["type"] == "http.response.start":
+                    headers = list(message.get("headers", []))
+                    # clients shouldn't care about the patch release
+                    version = __version__.rsplit('.', 1)[0]
+                    headers.append((b"DRYCC_API_VERSION", version.encode()))
+                    headers.append((b"DRYCC_PLATFORM_VERSION", __version__.encode()))
+                    message = message.copy()
+                    message["headers"] = headers
+                await send(message)
+            return await super().__call__(scope, receive, send_with_version)
+        else:
+            return await super().__call__(scope, receive, send)
+
+
 class ChannelOAuthMiddleware(BaseMiddleware):
     """
     Middleware which populates scope["user"] from a auth2 token.
@@ -39,7 +63,7 @@ class ChannelOAuthMiddleware(BaseMiddleware):
     authentication = DryccAuthentication()
 
     @database_sync_to_async
-    def get_user(self, scope):
+    def login(self, scope):
         headers = {}
         for header in scope["headers"]:
             if header[0] in (b"authorization", ):
@@ -49,10 +73,10 @@ def get_user(self, scope):
             return None
         request = HttpRequest()
         request.META = headers
-        user, _ = self.authentication.authenticate(request)
-        return user
+        scope["user"], _ = self.authentication.authenticate(request)
+        return scope["user"]
 
     async def __call__(self, scope, receive, send):
         scope = dict(scope)
-        scope["user"] = await self.get_user(scope)
+        await self.login(scope)
         return await super().__call__(scope, receive, send)
diff --git a/rootfs/api/routing.py b/rootfs/api/routing.py
index c37159207..6806b2fa0 100644
--- a/rootfs/api/routing.py
+++ b/rootfs/api/routing.py
@@ -3,11 +3,17 @@
 
 from . import consumers
 
+http_urlpatterns = [
+    re_path(
+        r'^v2/apps/(?P<id>([\w-]*))/volumes/(?P<name>([\w-]*))/filer/(?P<path>.*)$',
+        consumers.FilerProxyConsumer.as_asgi()),
+]
+
 websocket_urlpatterns = [
     re_path(
-        r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<pod_id>([\w-]*))/logs/?$',
+        r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<name>([\w-]*))/logs/?$',
         consumers.AppPodLogsConsumer.as_asgi()),
     re_path(
-        r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<pod_id>([\w-]*))/exec/?$',
+        r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<name>([\w-]*))/exec/?$',
         consumers.AppPodExecConsumer.as_asgi()),
 ]
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 9b4dafc86..d570e8726 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -97,7 +97,7 @@ def randstr(k):
     'django.middleware.common.CommonMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
-    'api.middleware.APIVersionMiddleware',
+    'api.middleware.DjangoAPIVersionMiddleware',
 ]
 
 ROOT_URLCONF = 'drycc.urls'
@@ -178,7 +178,15 @@ def randstr(k):
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'PAGE_SIZE': 100,
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',
-    'EXCEPTION_HANDLER': 'api.exceptions.custom_exception_handler'
+    'EXCEPTION_HANDLER': 'api.exceptions.custom_exception_handler',
+    'DEFAULT_THROTTLE_CLASSES': [
+        'rest_framework.throttling.AnonRateThrottle',
+        'rest_framework.throttling.UserRateThrottle'
+    ],
+    'DEFAULT_THROTTLE_RATES': {
+        'anon': os.environ.get('DRYCC_CONTROLLER_THROTTLE_RATES_ANON', '30/min'),
+        'user': os.environ.get('DRYCC_CONTROLLER_THROTTLE_RATES_USER', '60/min'),
+    }
 }
 
 # URLs that end with slashes are ugly
@@ -342,7 +350,6 @@ def randstr(k):
 DRYCC_FILER_IMAGE = os.environ.get('DRYCC_FILER_IMAGE', 'registry.drycc.cc/drycc/filer:canary')
 DRYCC_FILER_IMAGE_PULL_POLICY = os.environ.get('DRYCC_FILER_IMAGE_PULL_POLICY', 'IfNotPresent')
 DRYCC_FILER_DURATION = int(os.environ.get('DRYCC_FILER_DURATION', '3600'))
-DRYCC_FILER_WAITTIME = int(os.environ.get('DRYCC_FILER_WAITTIME', '1200'))
 
 # Define a global default on how many pods to bring up and then
 # take down sequentially during a deploy
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index e601ffff0..25abe6502 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -4,9 +4,13 @@
 import multiprocessing
 from django.core import signals
 from api.settings.celery import app
-from api.settings.production import DATABASES
+from api.settings.production import DATABASES, REST_FRAMEWORK
 from api.settings.production import *  # noqa
 
+
+# Disable DRF throttling for unit tests
+del REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']
+del REST_FRAMEWORK['DEFAULT_THROTTLE_CLASSES']
 # Fix Django test error.
 # https://github.com/django/django/blob/main/django/test/runner.py#L455
 # This code was removed in Django > 5.2, so it is safe to remove this when we upgrade.
diff --git a/rootfs/api/tests/test_consumers.py b/rootfs/api/tests/test_consumers.py
new file mode 100644
index 000000000..244edb3e8
--- /dev/null
+++ b/rootfs/api/tests/test_consumers.py
@@ -0,0 +1,932 @@
+"""
+Unit tests for the Drycc api consumers.
+
+Run the tests with "./manage.py test api.tests.test_consumers"
+"""
+import json
+import asyncio
+import aiohttp
+from unittest import mock
+from unittest.mock import MagicMock, patch
+
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from channels.exceptions import DenyConnection
+
+from api.models.app import App
+from api.models.volume import Volume
+from api.consumers import (
+    AppPermChecker,
+    BaseAppConsumer,
+    BaseK8sConsumer,
+    AppPodLogsConsumer,
+    AppPodExecConsumer,
+    FilerProxyConsumer
+)
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class ConsumerTestCase(DryccTransactionTestCase):
+    """Base test case for consumer tests"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.app_id = self.create_app()
+        self.app = App.objects.get(id=self.app_id)
+
+    def tearDown(self):
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+
+class AppPermCheckerTest(ConsumerTestCase):
+    """Tests for AppPermChecker"""
+
+    def test_has_perm_with_logged_in_user(self):
+        """Test permission check with valid user and app"""
+        checker = AppPermChecker({
+            'user': self.user,
+            'url_route': {'kwargs': {'id': self.app_id}}
+        })
+
+        # Mock the async method
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertTrue(result[0])  # Should have permission
+            # When permission is granted, message is None
+
+        asyncio.run(test_async())
+
+    def test_has_perm_with_anonymous_user(self):
+        """Test permission check with anonymous user"""
+        checker = AppPermChecker({
+            'user': None,
+            'url_route': {'kwargs': {'id': self.app_id}}
+        })
+
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertFalse(result[0])  # Should not have permission
+            self.assertEqual(result[1], "user not login")
+
+        asyncio.run(test_async())
+
+    def test_has_perm_with_nonexistent_app(self):
+        """Test permission check with non-existent app"""
+        checker = AppPermChecker({
+            'user': self.user,
+            'url_route': {'kwargs': {'id': 'nonexistent-app'}}
+        })
+
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertFalse(result[0])  # Should not have permission
+            self.assertEqual(result[1], "app not exists")
+
+        asyncio.run(test_async())
+
+    def test_has_perm_with_cache(self):
+        """Test permission check with cached permissions"""
+        checker = AppPermChecker({
+            'user': self.user,
+            'url_route': {'kwargs': {'id': self.app_id}}
+        })
+
+        # Set cache
+        cache_key = f"permission:user:{self.user.id}:app:{self.app_id}"
+        cache.set(cache_key, (True, "cached permission"))
+
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertTrue(result[0])
+            self.assertEqual(result[1], "cached permission")
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class BaseAppConsumerTest(ConsumerTestCase):
+    """Tests for BaseAppConsumer"""
+
+    def test_connect_with_permission(self, mock_requests):
+        """Test WebSocket connection with valid permissions"""
+        async def test_async():
+            # Mock the has_perm method to return True
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(BaseAppConsumer, 'accept') as mock_accept:
+                    consumer = BaseAppConsumer()
+                    consumer.scope = {
+                        'user': self.user,
+                        'url_route': {'kwargs': {'id': self.app_id}}
+                    }
+
+                    await consumer.connect()
+                    mock_accept.assert_called_once()
+
+        asyncio.run(test_async())
+
+    def test_connect_without_permission(self, mock_requests):
+        """Test WebSocket connection without valid permissions"""
+        async def test_async():
+            # Mock the has_perm method to return False
+            async def mock_has_perm():
+                return (False, "permission denied")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                consumer = BaseAppConsumer()
+                consumer.scope = {
+                    'user': self.user,
+                    'url_route': {'kwargs': {'id': self.app_id}}
+                }
+
+                with self.assertRaises(DenyConnection):
+                    await consumer.connect()
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class BaseK8sConsumerTest(ConsumerTestCase):
+    """Tests for BaseK8sConsumer"""
+
+    def test_kubernetes_property(self, mock_requests):
+        """Test kubernetes property initialization"""
+        consumer = BaseK8sConsumer()
+
+        # Mock the token file reading and Kubernetes API client
+        with patch('builtins.open', mock.mock_open(read_data='mock-token')):
+            with patch('api.consumers.Configuration') as mock_config:
+                with patch('api.consumers.core_v1_api.CoreV1Api') as mock_api:
+                    mock_config_instance = MagicMock()
+                    mock_config.return_value = mock_config_instance
+
+                    consumer.kubernetes
+
+                    # Verify configuration was called
+                    mock_config.assert_called_once()
+                    mock_api.assert_called_once()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class AppPodLogsConsumerTest(ConsumerTestCase):
+    """Tests for AppPodLogsConsumer"""
+
+    def test_connect(self, mock_requests):
+        """Test WebSocket connection for pod logs"""
+        async def test_async():
+            with patch.object(BaseAppConsumer, 'connect') as mock_super_connect:
+                consumer = AppPodLogsConsumer()
+                consumer.scope = {
+                    'user': self.user,
+                    'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+                }
+
+                await consumer.connect()
+
+                mock_super_connect.assert_called_once()
+                self.assertFalse(consumer.running)
+                self.assertIsNone(consumer.response)
+                self.assertTrue(consumer.conneted)
+                self.assertEqual(consumer.buffer, b'')
+                self.assertEqual(consumer.pod_name, 'test-pod')
+
+        asyncio.run(test_async())
+
+    def test_receive_log_request(self, mock_requests):
+        """Test receiving log request message"""
+        async def test_async():
+            consumer = AppPodLogsConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+            }
+            consumer.running = False
+            consumer.id = self.app_id
+            consumer.pod_name = 'test-pod'
+
+            # Mock kubernetes client
+            mock_k8s = MagicMock()
+            mock_response = MagicMock()
+            mock_k8s.read_namespaced_pod_log.return_value = mock_response
+
+            with patch.object(type(consumer), 'kubernetes', new_callable=lambda: mock_k8s):
+                # Create a proper async function instead of using AsyncMock
+                async def mock_sync_to_async_func(*args, **kwargs):
+                    return mock_response
+
+                with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                    mock_sync_to_async.return_value = mock_sync_to_async_func
+                    with patch.object(consumer, 'send'):
+                        with patch.object(consumer, 'close'):
+
+                            test_data = {
+                                "lines": 100,
+                                "follow": False,
+                                "previous": False,
+                                "container": "web"
+                            }
+
+                            await consumer.receive(text_data=json.dumps(test_data))
+
+                            self.assertTrue(consumer.running)
+
+        asyncio.run(test_async())
+
+    def test_disconnect(self, mock_requests):
+        """Test WebSocket disconnection"""
+        async def test_async():
+            consumer = AppPodLogsConsumer()
+            consumer.conneted = True
+            consumer.response = MagicMock()
+            consumer.response.connection.sock = MagicMock()
+
+            with patch('asyncio.get_event_loop') as mock_loop:
+                mock_event_loop = MagicMock()
+                mock_loop.return_value = mock_event_loop
+                with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_sync_close():
+                        pass
+                    mock_sync_to_async.return_value = mock_sync_close
+
+                    await consumer.disconnect(1000)
+
+                    self.assertFalse(consumer.conneted)
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class AppPodExecConsumerTest(ConsumerTestCase):
+    """Tests for AppPodExecConsumer"""
+
+    def test_connect(self, mock_requests):
+        """Test WebSocket connection for pod exec"""
+        async def test_async():
+            with patch.object(BaseAppConsumer, 'connect') as mock_super_connect:
+                consumer = AppPodExecConsumer()
+                consumer.scope = {
+                    'user': self.user,
+                    'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+                }
+                await consumer.connect()
+                mock_super_connect.assert_called_once()
+                self.assertIsNone(consumer.stream)
+                self.assertTrue(consumer.conneted)
+                self.assertEqual(consumer.pod_name, 'test-pod')
+
+        asyncio.run(test_async())
+
+    def test_send_stdout(self, mock_requests):
+        """Test sending stdout data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+
+            with patch.object(consumer.__class__.__bases__[0], 'send') as mock_super_send:
+                # Test sending bytes data
+                await consumer.send(b'test output', channel=1)  # STDOUT_CHANNEL
+                mock_super_send.assert_called_with(bytes_data=b'\x01test output')
+
+                # Test sending string data
+                await consumer.send('test string', channel=1)
+                mock_super_send.assert_called_with(text_data='\x01test string')
+
+        asyncio.run(test_async())
+
+    def test_send_none_data(self, mock_requests):
+        """Test sending None data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+
+            with patch.object(consumer.__class__.__bases__[0], 'send') as mock_super_send:
+                await consumer.send(None)
+                mock_super_send.assert_not_called()
+
+        asyncio.run(test_async())
+
+    def test_receive_exec_request(self, mock_requests):
+        """Test receiving exec request"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+            }
+            consumer.stream = None
+            consumer.pod_name = 'test-pod'
+            consumer.id = self.app_id
+
+            exec_request = {
+                "command": ["/bin/bash"],
+                "stdin": True,
+                "tty": True
+            }
+
+            # Mock the kubernetes property
+            mock_k8s = MagicMock()
+            mock_k8s.connect_get_namespaced_pod_exec = MagicMock()
+
+            with patch.object(type(consumer), 'kubernetes', new_callable=lambda: mock_k8s):
+                with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                    with patch('api.consumers.stream'):
+                        mock_stream_obj = MagicMock()
+                        # Create a proper async function instead of AsyncMock
+
+                        async def mock_sync_to_async_func(*args, **kwargs):
+                            return mock_stream_obj
+                        mock_sync_to_async.return_value = mock_sync_to_async_func
+
+                        # Mock the task method to avoid creating unawaited coroutine
+                        async def mock_task():
+                            pass
+                        with patch.object(
+                            consumer, 'task', side_effect=mock_task
+                        ) as mock_task_method:
+                            await consumer.receive(text_data=json.dumps(exec_request))
+                            self.assertEqual(consumer.stream, mock_stream_obj)
+                            mock_task_method.assert_called_once()
+
+        asyncio.run(test_async())
+
+    def test_receive_command_data(self, mock_requests):
+        """Test receiving command input data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            mock_stream = MagicMock()
+            consumer.stream = mock_stream
+
+            with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                # Create a proper async function instead of AsyncMock
+                async def mock_write_stdin(channel, data):
+                    pass
+                mock_sync_to_async.return_value = mock_write_stdin
+
+                # Test text data
+                await consumer.receive(text_data='\x00test input')
+                mock_sync_to_async.assert_called()
+
+        asyncio.run(test_async())
+
+    def test_receive_invalid_operation(self, mock_requests):
+        """Test receiving data when stream is None and no valid text data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            consumer.stream = None
+
+            with self.assertRaises(ValueError) as context:
+                await consumer.receive(bytes_data=b'invalid')
+
+            self.assertEqual(str(context.exception), "This operation is not supported!")
+
+        asyncio.run(test_async())
+
+    def test_disconnect(self, mock_requests):
+        """Test WebSocket disconnection"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            consumer.stream = MagicMock()
+            consumer.conneted = True
+
+            with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                # Create a proper async function instead of AsyncMock
+                async def mock_close_stream():
+                    pass
+                mock_sync_to_async.return_value = mock_close_stream
+
+                await consumer.disconnect(1000)
+
+                self.assertFalse(consumer.conneted)
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class FilerProxyConsumerTest(ConsumerTestCase):
+    """Tests for FilerProxyConsumer"""
+
+    def setUp(self):
+        super().setUp()
+        # Create a test volume using Volume.objects.create to avoid k8s calls
+        # We'll patch the save method to prevent k8s interactions
+        with patch.object(Volume, 'save_to_k8s'), patch.object(Volume, 'delete_from_k8s'):
+            self.test_volume = Volume.objects.create(
+                owner=self.user,
+                app=self.app,
+                name='test-volume',
+                size='1G',
+                path={'web': '/data'},
+                type='csi',
+                parameters={'csi': {'driver': 'test-driver'}}
+            )
+
+    def test_handle_without_permission(self, mock_requests):
+        """Test handling request without permission"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/_/ping',
+                'url_route': {
+                    'kwargs': {'path': '_/ping', 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            async def mock_has_perm():
+                return (False, "permission denied")
+
+            async def mock_bind_func():
+                return {'endpoint': '', 'username': 'test', 'password': 'test'}
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+                        await consumer.handle(b'')
+                        mock_send_response_obj.assert_awaited_once_with(403, b'permission denied')
+
+        asyncio.run(test_async())
+
+    def test_handle_ping_endpoint(self, mock_requests):
+        """Test handling ping endpoint"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/_/ping',
+                'url_route': {
+                    'kwargs': {'path': '_/ping', 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return {'username': 'test', 'password': 'test'}
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.info = mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'')
+
+                        mock_send_response_obj.assert_called_once_with(200, b"pong")
+
+        asyncio.run(test_async())
+
+    def test_handle_bind_endpoint(self, mock_requests):
+        """Test handling bind endpoint"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/',
+                'url_route': {
+                    'kwargs': {'path': '_/bind', 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            filer_data = {
+                'username': 'test-username',
+                'password': 'test-password'
+            }
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return filer_data
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'')
+
+                        expected_response = json.dumps({
+                            "username": "test-username",
+                            "password": "test-password"
+                        }).encode('utf-8')
+
+                        mock_send_response_obj.assert_called_once_with(200, expected_response)
+
+        asyncio.run(test_async())
+
+    def test_handle_proxy_request(self, mock_requests):
+        """Test handling proxy request to filer"""
+        path = 'webdav/files/test.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/{path}',
+                'url_route': {
+                    'kwargs': {'path': path, 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': [(b'host', b'example.com'), (b'user-agent', b'test-agent')],
+                'query_string': 'param=value'
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            filer_data = {
+                'endpoint': 'http://filer.example.com',
+                'username': 'test-username',
+                'password': 'test-password'
+            }
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(consumer, '_handle_proxy_request') as mock_handle_proxy:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return filer_data
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.info = mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'test body')
+
+                        expected_url = "http://filer.example.com{}?param=value".format(
+                            consumer.scope['path']
+                        )
+                        expected_headers = {'user-agent': 'test-agent'}
+
+                        mock_handle_proxy.assert_called_once_with(
+                            expected_url,
+                            'GET',
+                            expected_headers,
+                            b'test body'
+                        )
+
+        asyncio.run(test_async())
+
+    def test_forward_response(self, mock_requests):
+        """Test forwarding response from filer"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.config = MagicMock()
+            consumer.config.chunk_size = 1024
+
+            # Mock aiohttp response
+            mock_response = MagicMock()
+            mock_response.status = 200
+            mock_response.headers = {'content-type': 'application/json', 'content-length': '100'}
+
+            # Create an async iterator for iter_chunked
+            class AsyncIterator:
+                def __init__(self, items):
+                    self.items = iter(items)
+
+                def __aiter__(self):
+                    return self
+
+                async def __anext__(self):
+                    try:
+                        return next(self.items)
+                    except StopIteration:
+                        raise StopAsyncIteration
+
+            # Mock the iter_chunked method to return our async iterator
+            mock_response.content.iter_chunked = MagicMock(
+                return_value=AsyncIterator([b'chunk1', b'chunk2', b''])
+            )
+
+            # Mock async methods properly
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            async def mock_send_body(*args, **kwargs):
+                pass
+
+            async def mock_send_headers(*args, **kwargs):
+                pass
+
+            with patch.object(
+                consumer, 'send_body', side_effect=mock_send_body
+            ) as mock_send_body_obj:
+                with patch.object(
+                    consumer, 'send_headers', side_effect=mock_send_headers
+                ) as mock_send_headers_obj:
+                    await consumer._forward_response(mock_response)
+                    # Check that response headers were sent
+                    expected_headers = [
+                        [b'content-type', b'application/json'],
+                        [b'content-length', b'100']
+                    ]
+                    mock_send_headers_obj.assert_called_once_with(
+                        status=200,
+                        headers=expected_headers
+                    )
+                    # Check that body chunks were sent
+                    self.assertEqual(mock_send_body_obj.call_count, 3)
+
+        asyncio.run(test_async())
+
+    def test_handle_proxy_request_connection_error(self, mock_requests):
+        """Test handling connection error during proxy request"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.config = MagicMock()
+            consumer.config.timeout = aiohttp.ClientTimeout(total=30)
+
+            # Mock the entire session context manager flow
+            mock_session_instance = MagicMock()
+
+            async def mock_aenter(self):
+                return mock_session_instance
+
+            async def mock_aexit(self, *args):
+                return None
+            mock_session_instance.__aenter__ = mock_aenter
+            mock_session_instance.__aexit__ = mock_aexit
+
+            # Mock the request context manager to raise an exception
+            mock_request_cm = MagicMock()
+
+            async def mock_request_aenter(self):
+                raise aiohttp.ClientError("Connection failed")
+
+            async def mock_request_aexit(self, *args):
+                return None
+            mock_request_cm.__aenter__ = mock_request_aenter
+            mock_request_cm.__aexit__ = mock_request_aexit
+            mock_session_instance.request = MagicMock(return_value=mock_request_cm)
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch('aiohttp.ClientSession', return_value=mock_session_instance):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ):
+                    await consumer._handle_proxy_request(
+                        'http://example.com/test',
+                        'GET',
+                        {},
+                        b'proxy service unavailable: Connection failed'
+                    )
+
+        asyncio.run(test_async())
+
+    def test_handle_proxy_request_timeout(self, mock_requests):
+        """Test handling timeout during proxy request"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.config = MagicMock()
+            consumer.config.timeout = aiohttp.ClientTimeout(total=30)
+
+            # Mock the entire session context manager flow
+            mock_session_instance = MagicMock()
+
+            async def mock_aenter(self):
+                return mock_session_instance
+
+            async def mock_aexit(self, *args):
+                return None
+            mock_session_instance.__aenter__ = mock_aenter
+            mock_session_instance.__aexit__ = mock_aexit
+
+            # Mock the request context manager to raise a timeout
+            mock_request_cm = MagicMock()
+
+            async def mock_request_aenter(self):
+                raise asyncio.TimeoutError()
+
+            async def mock_request_aexit(self, *args):
+                return None
+            mock_request_cm.__aenter__ = mock_request_aenter
+            mock_request_cm.__aexit__ = mock_request_aexit
+            mock_session_instance.request = MagicMock(return_value=mock_request_cm)
+
+            with patch('aiohttp.ClientSession', return_value=mock_session_instance):
+                with patch.object(consumer, 'send_response') as mock_send_response:
+                    await consumer._handle_proxy_request(
+                        'http://example.com/test',
+                        'GET',
+                        {},
+                        b''
+                    )
+
+                    mock_send_response.assert_called_once_with(
+                        504,
+                        b'proxy request to backend filer timeout'
+                    )
+
+        asyncio.run(test_async())
+
+    def test_skip_request_headers(self, mock_requests):
+        """Test that certain headers are skipped in proxy requests"""
+        consumer = FilerProxyConsumer()
+
+        # Verify that SKIP_REQUEST_HEADERS contains expected headers
+        expected_skip_headers = {
+            'host', 'connection', 'keep-alive', 'proxy-connection',
+            'te', 'trailers', 'upgrade'
+        }
+
+        self.assertEqual(consumer.SKIP_REQUEST_HEADERS, expected_skip_headers)
+
+    def test_skip_response_headers(self, mock_requests):
+        """Test that certain headers are skipped in proxy responses"""
+        consumer = FilerProxyConsumer()
+
+        # Verify that SKIP_RESPONSE_HEADERS contains expected headers
+        expected_skip_headers = {
+            'connection', 'keep-alive', 'proxy-authenticate',
+            'proxy-authorization', 'te', 'trailers',
+            'transfer-encoding', 'upgrade'
+        }
+
+        self.assertEqual(consumer.SKIP_RESPONSE_HEADERS, expected_skip_headers)
+
+    def test_get_client_with_existing_volume(self, mock_requests):
+        """Test _get_client method with existing volume"""
+        path = 'test/file.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {
+                    'kwargs': {'path': path, 'name': 'test-volume', 'id': self.app_id}
+                }
+            }
+            consumer.id = self.app_id
+
+            with patch('api.filer.FilerClient') as mock_filer_client:
+                mock_client = MagicMock()
+                mock_filer_client.return_value = mock_client
+                # Test the _get_client method
+                client = await consumer._get_client(path)
+                # Verify that FilerClient was called with correct parameters
+                mock_filer_client.assert_called_once_with(self.app_id, self.test_volume, path)
+                self.assertEqual(client, mock_client)
+
+        asyncio.run(test_async())
+
+    def test_get_client_with_nonexistent_volume(self, mock_requests):
+        """Test _get_client method with non-existent volume"""
+        path = 'test/file.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {
+                    'kwargs': {
+                        'path': path, 'name': 'nonexistent-volume', 'id': self.app_id,
+                    }
+                }
+            }
+            consumer.id = self.app_id
+            # Test the _get_client method
+            client = await consumer._get_client(path)
+            # Should return None when volume doesn't exist
+            self.assertIsNone(client)
+
+        asyncio.run(test_async())
+
+    def test_handle_with_volume_not_found(self, mock_requests):
+        """Test handling request when volume is not found"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            path = 'test/file.txt'
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/nonexistent-volume/filer/{path}',
+                'url_route': {
+                    'kwargs': {
+                        'path': path, 'name': 'nonexistent-volume', 'id': self.app_id,
+                    }
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            consumer.id = self.app_id
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    await consumer.handle(b'test body')
+                    mock_send_response_obj.assert_called_once_with(
+                        status=404,
+                        body=b'app or volume not found'
+                    )
+
+        asyncio.run(test_async())
+
+    def test_handle_with_existing_volume(self, mock_requests):
+        """Test handling request with existing volume"""
+        path = 'webdav/test/file.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/nonexistent-volume/filer/{path}',
+                'url_route': {
+                    'kwargs': {'path': path, 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': [(b'host', b'example.com'), (b'user-agent', b'test-agent')],
+                'query_string': 'param=value'
+            }
+            consumer.id = self.app_id
+
+            filer_data = {
+                'endpoint': 'http://filer.example.com',
+                'username': 'test-username',
+                'password': 'test-password'
+            }
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(consumer, '_handle_proxy_request') as mock_handle_proxy:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return filer_data
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.bind = mock_client.info = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'test body')
+
+                        expected_url = "http://filer.example.com{}?param=value".format(
+                            consumer.scope['path']
+                        )
+                        expected_headers = {'user-agent': 'test-agent'}
+
+                        mock_handle_proxy.assert_called_once_with(
+                            expected_url,
+                            'GET',
+                            expected_headers,
+                            b'test body'
+                        )
+
+        asyncio.run(test_async())
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 85199e622..647b119fb 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -118,15 +118,6 @@
         r"^apps/(?P<id>{})/volumes/(?P<name>{})/path/?$".format(
             settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppVolumesViewSet.as_view({'patch': 'path'})),
-    # application filer
-    re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>{})/client/?$".format(
-            settings.APP_URL_REGEX, settings.NAME_REGEX),
-        views.AppFilerClientViewSet.as_view({'get': 'list', 'post': 'create'})),
-    re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>{})/client/(?P<path>[\S]+)$".format(
-            settings.APP_URL_REGEX, settings.NAME_REGEX),
-        views.AppFilerClientViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
     # application resources
     re_path(r"^resources/services/?$", views.AppResourcesViewSet.as_view({'get': 'services'})),
     re_path(
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index ecfa8b4f8..ca81b8508 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -41,15 +41,15 @@
 from django.views.decorators.cache import never_cache
 from django.contrib.auth import REDIRECT_FIELD_NAME
 from django.views.decorators.csrf import csrf_exempt
-from django.http.response import FileResponse, JsonResponse, StreamingHttpResponse
+from django.http.response import JsonResponse, StreamingHttpResponse
 from channels.db import database_sync_to_async
 from social_django.utils import psa
 from social_django.views import _do_login
 from social_core.utils import setting_name
-from api import admissions, utils, filer
+from api import admissions, utils
 from api.backend import OauthCacheManager
 from api.apps_extra.social_core.actions import do_auth, do_complete
-from api.files.parsers import FilerUploadParser
+
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -1002,51 +1002,6 @@ def path(self, request, *args, **kwargs):
         return Response(serializer.data)
 
 
-class AppFilerClientViewSet(AppFilterViewSet):
-    """RESTful views for volumes apps with collaborators."""
-    model = models.volume.Volume
-    parser_classes = [FilerUploadParser]
-
-    def get_client(self):
-        volume = get_object_or_404(
-            models.volume.Volume, app=self.get_app(), name=self.kwargs['name'])
-        return filer.FilerClient(volume.app.id, volume, volume.app.scheduler)
-
-    def list(self, request, **kwargs):
-        path = request.query_params.get('path', '')
-        client = self.get_client()
-        response = client.get(path, params={"action": "list"})
-        if response.status_code == 200:
-            results = response.json()
-            # fake out pagination for now
-            pagination = {'results': results, 'count': len(results)}
-            return Response(data=pagination)
-        return Response(status=response.status_code, data=response.text)
-
-    def retrieve(self, request, **kwargs):
-        path = self.kwargs.get('path', '')
-        client = self.get_client()
-        chunk_size = 1024 * 1024 * 2
-        response = client.get(path, stream=True, params={"action": "get"})
-        return FileResponse(
-            status=response.status_code,
-            headers=response.headers,
-            streaming_content=utils.iter_to_aiter(response.iter_content(chunk_size=chunk_size)),
-        )
-
-    def create(self, request, **kwargs):
-        client = self.get_client()
-        file = request.data['file']
-        response = client.post(file.filepath, files=request.FILES)
-        return Response(data=response.content, status=response.status_code)
-
-    def destroy(self, request, **kwargs):
-        path = self.kwargs.get('path', '')
-        client = self.get_client()
-        response = client.delete(path)
-        return Response(data=response.content, status=response.status_code)
-
-
 class AppResourcesViewSet(AppFilterViewSet):
     """RESTful views for resources apps with collaborators."""
     model = models.resource.Resource
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index b39a97da9..716fc0107 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,7 +1,7 @@
 # Drycc controller requirements
 inotify==0.2.12
 backoff==2.2.1
-django==5.2.8
+django==5.2.9
 channels==4.3.1
 aiohttp==3.13.2
 django-cors-headers==4.9.0

From dcb51ca744fd58e05d0b872932f9975d90be7956 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 5 Dec 2025 16:24:35 +0800
Subject: [PATCH 191/200] feat(dryccfile): merge config into dryccfile

---
 rootfs/api/models/build.py      | 21 +++++++++
 rootfs/api/tests/test_config.py | 78 +++++++++++++++++++++++++++++++++
 rootfs/api/views.py             |  4 +-
 3 files changed, 101 insertions(+), 2 deletions(-)

diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index cbd097523..2b91d755c 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -1,4 +1,6 @@
 import logging
+from copy import deepcopy
+from collections import defaultdict
 from django.db import models
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException
@@ -63,6 +65,25 @@ def source_based(self):
     def version(self):
         return 'git-{}'.format(self.sha) if self.source_based else 'latest'
 
+    def merge(self, config):
+        if self.dryccfile and any({"values", "values_refs"} & set(config.diff(None).keys())):
+            dryccfile = deepcopy(self.dryccfile)
+            ptype_env, group_env = defaultdict(dict), defaultdict(dict)
+            for value in config.values:
+                if "ptype" in value:
+                    ptype_env[value["ptype"]][value["name"]] = value["value"]
+                if "group" in value:
+                    group_env[value["group"]][value["name"]] = value["value"]
+            dryccfile['config'] = dict(group_env)
+            for pipeline in dryccfile['pipeline'].values():
+                pipeline['env'] = ptype_env[pipeline['ptype']]
+                pipeline['config'] = config.values_refs.get(pipeline['ptype'], [])
+            return Build.objects.create(
+                owner=config.owner, app=self.app, image=self.image, stack=self.stack, sha=self.sha,
+                procfile=self.procfile, dryccfile=dryccfile, dockerfile=self.dockerfile,
+            )
+        return self
+
     def get_image(self, ptype, default_image=None):
         pipeline = self.get_pipeline(ptype)
         if pipeline and 'build' in pipeline:
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index b9d85c776..ba4a62064 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -1034,3 +1034,81 @@ def test_config_from_dryccfile(self, mock_requests):
         self.assertEqual(release.failed, False)
         self.assertEqual(release.config.envs("web"), {})
         self.assertEqual(release.config.values_refs, {})
+
+        # test config modification creates new build with merged dryccfile
+        # First, create a build with dryccfile
+        build_body_with_dryccfile = copy.deepcopy(build_body)
+        build_body_with_dryccfile['dryccfile']['config'] = {
+            "mygroup1": {
+                "GROUP": "g1",
+                "DEBUG": "tr",
+            }
+        }
+        build_body_with_dryccfile['dryccfile']['pipeline']['web.yaml']['env'] = {
+            "PENV1": "web"
+        }
+        build_body_with_dryccfile['dryccfile']['pipeline']['web.yaml']['config'] = [
+            "mygroup1"
+        ]
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body_with_dryccfile)
+            self.assertEqual(response.status_code, 201, response.data)
+
+        # Get the latest release and build
+        release = app.release_set.latest()
+        initial_build = release.build
+        self.assertIsNotNone(initial_build.dryccfile)
+
+        # Now modify config - this should trigger build.merge() and create a new build
+        config_values = [
+            {"name": "NEW_CONFIG", "value": "new_value", "group": "mygroup1"},
+            {"name": "PENV2", "value": "web2", "ptype": "web"}
+        ]
+        body = {'values': config_values}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # Check that a new release was created
+        new_release = app.release_set.latest()
+        self.assertNotEqual(release.uuid, new_release.uuid)
+
+        # Check that a new build was created with merged dryccfile
+        new_build = new_release.build
+        self.assertIsNotNone(new_build)
+        self.assertNotEqual(initial_build.uuid, new_build.uuid)
+
+        # Verify dryccfile was merged with config values
+        self.assertIn('dryccfile', new_build.__dict__)
+        dryccfile = new_build.dryccfile
+
+        # Check config groups were merged
+        self.assertIn('config', dryccfile)
+        self.assertIn('mygroup1', dryccfile['config'])
+        self.assertEqual(dryccfile['config']['mygroup1']['GROUP'], 'g1')
+        self.assertEqual(dryccfile['config']['mygroup1']['DEBUG'], 'tr')
+        self.assertEqual(dryccfile['config']['mygroup1']['NEW_CONFIG'], 'new_value')
+
+        # Check ptype envs were merged
+        self.assertIn('pipeline', dryccfile)
+        self.assertIn('web.yaml', dryccfile['pipeline'])
+        self.assertIn('env', dryccfile['pipeline']['web.yaml'])
+        self.assertEqual(dryccfile['pipeline']['web.yaml']['env']['PENV1'], 'web')
+        self.assertEqual(dryccfile['pipeline']['web.yaml']['env']['PENV2'], 'web2')
+
+        # Check config refs
+        self.assertIn('config', dryccfile['pipeline']['web.yaml'])
+        self.assertEqual(dryccfile['pipeline']['web.yaml']['config'], ['mygroup1'])
+
+        # Verify config values are correct
+        config = new_release.config
+        expected_envs = {
+            'GROUP': 'g1',
+            'DEBUG': 'tr',
+            'NEW_CONFIG': 'new_value',
+            'PENV1': 'web',
+            'PENV2': 'web2'
+        }
+        self.assertEqual(config.envs("web"), expected_envs)
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index ca81b8508..37126dc23 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -477,8 +477,8 @@ def delete(self, request, **kwargs):
     def post_save(self, config):
         latest_release = models.release.Release.latest(self.get_app())
         try:
-            release = latest_release.new(
-                self.request.user, config=config, build=latest_release.build)
+            build = latest_release.build.merge(config) if latest_release.build else None
+            release = latest_release.new(self.request.user, config=config, build=build)
             if release.build and config.app.appsettings_set.latest().autodeploy:
                 release.deploy(release.ptypes, False)
         except BaseException as e:

From 3846c3a4a5f9c413a29f3213f02a48018018881e Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 5 Dec 2025 18:04:17 +0800
Subject: [PATCH 192/200] chore(scheduler): optimize
 termination_grace_period_seconds parameter passing

---
 rootfs/api/models/app.py                    | 17 ++++++++-------
 rootfs/api/tests/test_app.py                | 23 ++++++++++++---------
 rootfs/scheduler/resources/pod.py           | 11 +---------
 rootfs/scheduler/tests/test_deployments.py  |  6 +++---
 rootfs/scheduler/tests/test_hpa.py          |  4 ++--
 rootfs/scheduler/tests/test_hpa_12_lower.py |  4 ++--
 rootfs/scheduler/tests/test_pods.py         |  2 +-
 rootfs/scheduler/tests/test_rc.py           |  4 ++--
 8 files changed, 34 insertions(+), 37 deletions(-)

diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index c58b61f15..c516cea5f 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1205,9 +1205,13 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
             settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT)
 
         # get application level pod termination grace period
-        pod_termination_grace_period_seconds = int(envs.get(
-            'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS',
-            settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS))
+        termination_grace_period_seconds = int(config.termination_grace_period.get(
+            ptype,
+            envs.get(
+                'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS',
+                settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS
+            )
+        ))
 
         # set the image pull policy that is associated with the application container
         image_pull_policy = envs.get('IMAGE_PULL_POLICY', settings.IMAGE_PULL_POLICY)
@@ -1240,16 +1244,15 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
             'healthcheck': healthcheck,
             'runtime_class_name': limit_plan.runtime_class_name,
             'dns_policy': settings.DRYCC_APP_DNS_POLICY,
-            'lifecycle_post_start': config.lifecycle_post_start,
-            'lifecycle_pre_stop': config.lifecycle_pre_stop,
+            'lifecycle_post_start': config.lifecycle_post_start.get(ptype, {}),
+            'lifecycle_pre_stop': config.lifecycle_pre_stop.get(ptype, {}),
             'routable': routable,
             'deploy_batches': batches,
             'restart_policy': "Always",
             'deploy_timeout': deploy_timeout,
             'deployment_revision_history_limit': deployment_history,
             'release_summary': release.summary,
-            'pod_termination_grace_period_seconds': pod_termination_grace_period_seconds,
-            'pod_termination_grace_period_each': config.termination_grace_period,
+            'termination_grace_period_seconds': termination_grace_period_seconds,
             'image_pull_secret_name': image_pull_secret_name,
             'image_pull_policy': image_pull_policy,
             'volumes': volumes,
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index f2ff92811..651156eed 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -628,25 +628,28 @@ def test_gather_app_settings(self, mock_requests):
         url = f"/v2/apps/{app.id}/build"
         response = self.client.post(url, data)
         self.assertEqual(response.status_code, 201, response.data)
-        Config.objects.create(
-            owner=self.user,
-            app=app,
-            values=[
+        # Set some app settings
+        url = f"/v2/apps/{app.id}/config"
+        body = {
+            'values': [
                 {"name": "DRYCC_DEPLOY_TIMEOUT", "value": "60", "group": "global"},
                 {"name": "DRYCC_DEPLOY_BATCHES", "value": "3", "group": "global"},
                 {
                     "name": "KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS",
-                    "value": "60", "group": "global"
+                    "value": "90", "group": "global"
                 },
-            ]
-        )
+            ],
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        # Gather app settings
         s = app._gather_app_settings(app.release_set.latest(),
                                      app.appsettings_set.latest(),
                                      'web',
                                      3)
-        assert isinstance(s['deploy_batches'], int)
-        assert isinstance(s['deploy_timeout'], int)
-        assert isinstance(s['pod_termination_grace_period_seconds'], int)
+        self.assertEqual(s['deploy_batches'], 3)
+        self.assertEqual(s['deploy_timeout'], 60)
+        self.assertEqual(s['termination_grace_period_seconds'], 90)
 
     def test_app_name_bad_regex(self, mock_requests):
         """
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 13af05245..50fbcc110 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -169,7 +169,7 @@ def manifest(self, namespace, name, image, **kwargs):
         spec['nodeSelector'].update(kwargs.get('tags', {}))
 
         # How long until a pod is forcefully terminated. 30 is kubernetes default
-        spec['terminationGracePeriodSeconds'] = self._get_termination_grace_period(kwargs)  # noqa
+        spec['terminationGracePeriodSeconds'] = kwargs.get('termination_grace_period_seconds', 30)
         # set pod volumes
         spec['volumes'] = kwargs.get('volumes', [])
         # create the base container
@@ -258,15 +258,6 @@ def _set_container_default_env(self, data):
             data["env"].append({"name": "DRYCC_DEBUG", "value": "1"})
         return data
 
-    @staticmethod
-    def _get_termination_grace_period(kwargs):
-        """ return termination grace period """
-        app_type = kwargs.get("app_type")
-        timeout_global = kwargs.get('pod_termination_grace_period_seconds', 30)
-        timeout_local = kwargs.get("pod_termination_grace_period_each", {}).get(app_type)
-
-        return timeout_global if timeout_local is None else int(timeout_local)
-
     @staticmethod
     def _get_memory(size):
         """ Format memory limit value """
diff --git a/rootfs/scheduler/tests/test_deployments.py b/rootfs/scheduler/tests/test_deployments.py
index fed4705c7..636e1ff64 100644
--- a/rootfs/scheduler/tests/test_deployments.py
+++ b/rootfs/scheduler/tests/test_deployments.py
@@ -24,7 +24,7 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
@@ -45,7 +45,7 @@ def update(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
@@ -67,7 +67,7 @@ def scale(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
diff --git a/rootfs/scheduler/tests/test_hpa.py b/rootfs/scheduler/tests/test_hpa.py
index 831adf47a..832d98e46 100644
--- a/rootfs/scheduler/tests/test_hpa.py
+++ b/rootfs/scheduler/tests/test_hpa.py
@@ -23,7 +23,7 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 1),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
@@ -72,7 +72,7 @@ def update_deployment(self, namespace=None, name=generate_random_name(), **kwarg
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
diff --git a/rootfs/scheduler/tests/test_hpa_12_lower.py b/rootfs/scheduler/tests/test_hpa_12_lower.py
index 1da518ee0..5cbca133e 100644
--- a/rootfs/scheduler/tests/test_hpa_12_lower.py
+++ b/rootfs/scheduler/tests/test_hpa_12_lower.py
@@ -27,7 +27,7 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 1),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
@@ -76,7 +76,7 @@ def update_deployment(self, namespace=None, name=generate_random_name(), **kwarg
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
diff --git a/rootfs/scheduler/tests/test_pods.py b/rootfs/scheduler/tests/test_pods.py
index 21105997f..b1373d6dd 100644
--- a/rootfs/scheduler/tests/test_pods.py
+++ b/rootfs/scheduler/tests/test_pods.py
@@ -25,7 +25,7 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
diff --git a/rootfs/scheduler/tests/test_rc.py b/rootfs/scheduler/tests/test_rc.py
index 117556cac..871f903b5 100644
--- a/rootfs/scheduler/tests/test_rc.py
+++ b/rootfs/scheduler/tests/test_rc.py
@@ -21,7 +21,7 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',
@@ -43,7 +43,7 @@ def scale_rc(self, namespace=None, name=generate_random_name(), **kwargs):
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
             'deploy_timeout': 120,
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
             'command': 'sh',
             'args': 'start',

From b621d3493b4ab7c2273b468d4d12f66ead4dc0a9 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Fri, 5 Dec 2025 23:35:44 +0800
Subject: [PATCH 193/200] chore(config): unify lifecycle interface

---
 ...le_post_start_config_lifecycle_and_more.py | 22 ++++++++++++
 rootfs/api/models/app.py                      |  6 ++--
 rootfs/api/models/config.py                   |  7 ++--
 rootfs/api/serializers/__init__.py            |  3 +-
 rootfs/api/serializers/schemas/healthcheck.py | 31 ++++++++--------
 rootfs/api/tests/test_config.py               |  6 ++--
 rootfs/scheduler/resources/pod.py             | 35 +------------------
 7 files changed, 46 insertions(+), 64 deletions(-)
 create mode 100644 rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py

diff --git a/rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py b/rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py
new file mode 100644
index 000000000..27cdfd3ff
--- /dev/null
+++ b/rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py
@@ -0,0 +1,22 @@
+# Generated by Django 5.2.8 on 2025-12-05 13:39
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0026_app_suspended_state'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='config',
+            old_name='lifecycle_post_start',
+            new_name='lifecycle',
+        ),
+        migrations.RemoveField(
+            model_name='config',
+            name='lifecycle_pre_stop',
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index c516cea5f..8da9a0d8b 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1227,7 +1227,6 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
         routable = True if (
             ptype == PTYPE_WEB and app_settings.routable) else False
 
-        healthcheck = config.healthcheck.get(ptype, {})
         volumes, volume_mounts = self._get_volumes_and_mounts(ptype, volumes)
         volumes.extend(limit_plan.pod_volumes)
         volume_mounts.extend(limit_plan.container_volume_mounts)
@@ -1241,11 +1240,10 @@ def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=N
             'resources': {"limits": limit_plan.limits, "requests": limit_plan.requests},
             'build_type': release.build.type,
             'annotations': limit_plan.annotations,
-            'healthcheck': healthcheck,
+            'lifecycle': config.lifecycle.get(ptype, {}),
+            'healthcheck': config.healthcheck.get(ptype, {}),
             'runtime_class_name': limit_plan.runtime_class_name,
             'dns_policy': settings.DRYCC_APP_DNS_POLICY,
-            'lifecycle_post_start': config.lifecycle_post_start.get(ptype, {}),
-            'lifecycle_pre_stop': config.lifecycle_pre_stop.get(ptype, {}),
             'routable': routable,
             'deploy_batches': batches,
             'restart_policy': "Always",
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index b298a60fd..649434b76 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -16,19 +16,18 @@ class Config(UuidAuditedModel):
     Set of configuration values applied as environment variables
     during runtime execution of the Application.
     """
-    ptype_fields = ("lifecycle_post_start", "lifecycle_pre_stop", "tags", "limits",
-                    "values_refs", "healthcheck", "termination_grace_period", "registry")
+    ptype_fields = ("lifecycle", "tags", "limits", "values_refs", "healthcheck",
+                    "termination_grace_period", "registry")
     allof_fields = ("values", ) + ptype_fields
 
     owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     values = models.JSONField(default=list, blank=True)
     values_refs = models.JSONField(default=dict, blank=True)
-    lifecycle_post_start = models.JSONField(default=dict, blank=True)
-    lifecycle_pre_stop = models.JSONField(default=dict, blank=True)
     tags = models.JSONField(default=dict, blank=True)
     limits = models.JSONField(default=dict, blank=True)
     registry = models.JSONField(default=dict, blank=True)
+    lifecycle = models.JSONField(default=dict, blank=True)
     healthcheck = models.JSONField(default=dict, blank=True)
     termination_grace_period = models.JSONField(default=dict, blank=True)
 
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 3fda7648c..a1a9ff754 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -280,8 +280,7 @@ class ConfigSerializer(serializers.ModelSerializer):
     owner = serializers.ReadOnlyField(source='owner.username')
     values = JSONFieldSerializer(required=False, binary=True)
     limits = JSONFieldSerializer(required=False, binary=True)
-    lifecycle_post_start = JSONFieldSerializer(required=False, binary=True)
-    lifecycle_pre_stop = JSONFieldSerializer(required=False, binary=True)
+    lifecycle = JSONFieldSerializer(required=False, binary=True)
     tags = JSONFieldSerializer(required=False, binary=True)
     registry = JSONFieldSerializer(required=False, binary=True)
     healthcheck = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
diff --git a/rootfs/api/serializers/schemas/healthcheck.py b/rootfs/api/serializers/schemas/healthcheck.py
index 8c6aa2d07..c4cc604ca 100644
--- a/rootfs/api/serializers/schemas/healthcheck.py
+++ b/rootfs/api/serializers/schemas/healthcheck.py
@@ -3,8 +3,6 @@
 
     "type": "object",
     "properties": {
-        # Exec specifies the action to take.
-        # More info: http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_execaction
         "exec": {
             "type": "object",
             "properties": {
@@ -16,8 +14,14 @@
             },
             "required": ["command"]
         },
-        # HTTPGet specifies the http request to perform.
-        # More info: http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_httpgetaction
+        "grpc": {
+            "type": "object",
+            "properties": {
+                "port": {"type": "integer"},
+                "service": {"type": "string"},
+            },
+            "required": ["port"]
+        },
         "httpGet": {
             "type": "object",
             "properties": {
@@ -39,8 +43,6 @@
             },
             "required": ["port"]
         },
-        # TCPSocket specifies an action involving a TCP port.
-        # More info: http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_tcpsocketaction
         "tcpSocket": {
             "type": "object",
             "properties": {
@@ -48,19 +50,16 @@
             },
             "required": ["port"]
         },
-        # Number of seconds after the container has started before liveness probes are initiated.
-        # More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
         "initialDelaySeconds": {"type": "integer"},
-        # Number of seconds after which the probe times out.
-        # More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
         "timeoutSeconds": {"type": "integer"},
-        # How often (in seconds) to perform the probe.
         "periodSeconds": {"type": "integer"},
-        # Minimum consecutive successes for the probe to be considered successful
-        # after having failed.
         "successThreshold": {"type": "integer"},
-        # Minimum consecutive failures for the probe to be considered
-        # failed after having succeeded.
         "failureThreshold": {"type": "integer"},
-    }
+    },
+    "oneOf": [
+        {"required": ["exec"]},
+        {"required": ["grpc"]},
+        {"required": ["httpGet"]},
+        {"required": ["tcpSocket"]},
+    ]
 }
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index ba4a62064..22a435fea 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -368,8 +368,7 @@ def test_response_data(self, mock_requests):
         for key in response.data:
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values',
                                 'values_refs', 'limits', 'tags', 'registry', 'healthcheck',
-                                'lifecycle_post_start', 'lifecycle_pre_stop',
-                                'termination_grace_period'])
+                                'lifecycle', 'termination_grace_period'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
@@ -400,8 +399,7 @@ def test_response_data_types_converted(self, mock_requests):
         for key in response.data:
             self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'limits',
                                 'values_refs', 'tags', 'registry', 'healthcheck',
-                                'lifecycle_post_start', 'lifecycle_pre_stop',
-                                'termination_grace_period'])
+                                'lifecycle', 'termination_grace_period'])
         expected = {
             'owner': self.user.username,
             'app': app_id,
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 50fbcc110..ec2ab4cdb 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -1,4 +1,3 @@
-from collections import defaultdict
 from datetime import datetime, timedelta, timezone
 import operator
 import os
@@ -236,8 +235,8 @@ def _set_container(self, namespace, container_name, data, **kwargs):
         self._set_container_default_env(data)
         # list sorted by dict key name
         data['env'].sort(key=operator.itemgetter('name'))
+        data['lifecycle'] = data.get('lifecycle', {})
         self._set_health_checks(data, env, **kwargs)
-        self._set_lifecycle_hooks(data, env, **kwargs)
 
     def _set_container_default_env(self, data):
         # set fields env
@@ -286,38 +285,6 @@ def _set_health_checks(self, container, env, **kwargs):
             container.update(
                 self._default_container_readiness_probe(env.get('PORT', DEFAULT_CONTAINER_PORT)))
 
-    @staticmethod
-    def _set_lifecycle_hooks(container, env, **kwargs):
-        app_type = kwargs.get("app_type")
-        lifecycle_post_start = kwargs.get('lifecycle_post_start', {})
-        lifecycle_post_start = lifecycle_post_start.get(app_type)
-        lifecycle_pre_stop = kwargs.get('lifecycle_pre_stop', {})
-        lifecycle_pre_stop = lifecycle_pre_stop.get(app_type)
-        if lifecycle_post_start or lifecycle_pre_stop:
-            lifecycle = defaultdict(dict)
-
-            if lifecycle_post_start:
-                lifecycle["postStart"] = {
-                        'exec': {
-                            "command": [
-                                "/bin/bash",
-                                "-c",
-                                "{0}".format(lifecycle_post_start)
-                            ]
-                        }
-                }
-            if lifecycle_pre_stop:
-                lifecycle["preStop"] = {
-                        'exec': {
-                            "command": [
-                                "/bin/bash",
-                                "-c",
-                                "{0}".format(lifecycle_pre_stop)
-                            ]
-                        }
-                }
-            container["lifecycle"] = dict(lifecycle)
-
     @staticmethod
     def _default_container_readiness_probe(port, delay=5, timeout=5, period_seconds=5,
                                            success_threshold=1, failure_threshold=1):

From ee1d37e889d6c24bb5ed8c7d2b55e6526844d2b8 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 11 Dec 2025 11:36:10 +0800
Subject: [PATCH 194/200] fix(lifecycle): add convert_to_str to serializers

---
 rootfs/api/serializers/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index a1a9ff754..6bd1e624f 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -280,7 +280,7 @@ class ConfigSerializer(serializers.ModelSerializer):
     owner = serializers.ReadOnlyField(source='owner.username')
     values = JSONFieldSerializer(required=False, binary=True)
     limits = JSONFieldSerializer(required=False, binary=True)
-    lifecycle = JSONFieldSerializer(required=False, binary=True)
+    lifecycle = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
     tags = JSONFieldSerializer(required=False, binary=True)
     registry = JSONFieldSerializer(required=False, binary=True)
     healthcheck = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)

From 8a82327e81d604e7408c60066258190c4603c8af Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 11 Dec 2025 13:08:09 +0800
Subject: [PATCH 195/200] fix(lifecycle): pod missing lifecycle field

---
 rootfs/scheduler/resources/pod.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index ec2ab4cdb..94de5cd80 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -235,7 +235,7 @@ def _set_container(self, namespace, container_name, data, **kwargs):
         self._set_container_default_env(data)
         # list sorted by dict key name
         data['env'].sort(key=operator.itemgetter('name'))
-        data['lifecycle'] = data.get('lifecycle', {})
+        data['lifecycle'] = kwargs.get('lifecycle', {})
         self._set_health_checks(data, env, **kwargs)
 
     def _set_container_default_env(self, data):

From 90da4cb344b67f2ad0ef7ddf488dc521fdbdc803 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 11 Jan 2026 21:10:03 +0800
Subject: [PATCH 196/200] chore(deps): bump aiohttp in /rootfs in the pip group
 across 1 directory (#201)

---
updated-dependencies:
- dependency-name: aiohttp
  dependency-version: 3.13.3
  dependency-type: direct:production
  dependency-group: pip
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 716fc0107..2a250d3d8 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -3,7 +3,7 @@ inotify==0.2.12
 backoff==2.2.1
 django==5.2.9
 channels==4.3.1
-aiohttp==3.13.2
+aiohttp==3.13.3
 django-cors-headers==4.9.0
 django-guardian==3.2.0
 djangorestframework==3.16.1

From 011bb39d77c7d4090e00ce9d649242e09119e181 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 15 Jan 2026 15:41:59 +0800
Subject: [PATCH 197/200] feat(ocid): use drycc ocid discover url

---
 rootfs/api/apps_extra/social_core/actions.py  |  4 +--
 .../social_core/backends.py}                  | 24 +++-------------
 .../api/apps_extra/social_core/pipelines.py   |  6 ++++
 rootfs/api/authentication.py                  |  4 +--
 rootfs/api/models/base.py                     |  2 +-
 rootfs/api/pipeline.py                        | 22 ---------------
 rootfs/api/settings/production.py             | 28 ++-----------------
 rootfs/api/views.py                           | 11 +++++---
 8 files changed, 25 insertions(+), 76 deletions(-)
 rename rootfs/api/{backend.py => apps_extra/social_core/backends.py} (75%)
 create mode 100644 rootfs/api/apps_extra/social_core/pipelines.py
 delete mode 100644 rootfs/api/pipeline.py

diff --git a/rootfs/api/apps_extra/social_core/actions.py b/rootfs/api/apps_extra/social_core/actions.py
index 2e37771a7..c86bac74b 100644
--- a/rootfs/api/apps_extra/social_core/actions.py
+++ b/rootfs/api/apps_extra/social_core/actions.py
@@ -39,7 +39,7 @@ def form2json(form_data):
         query = urlparse("?" + form_data).query
         params = parse_qs(query)
         return {key: params[key][0] for key in params}
-    from api.backend import OauthCacheManager
+    from api.apps_extra.social_core.backends import OauthCacheManager
     oauth_cache_manager = OauthCacheManager()
     oauth_cache_manager.set_state(data.get("key", ""), form2json(url).get("state"))
     return response
@@ -129,7 +129,7 @@ def do_complete(backend, login, user=None, redirect_name="next", *args, **kwargs
     if social_auth and social_auth.extra_data:
         extra_data = json.loads(social_auth.extra_data) if \
             isinstance(social_auth.extra_data, str) else social_auth.extra_data
-    from api.backend import OauthCacheManager
+    from api.apps_extra.social_core.backends import OauthCacheManager
     oauth_cache_manager = OauthCacheManager()
     oauth_cache_manager.set_token(data.get("state"), extra_data)
     return response
diff --git a/rootfs/api/backend.py b/rootfs/api/apps_extra/social_core/backends.py
similarity index 75%
rename from rootfs/api/backend.py
rename to rootfs/api/apps_extra/social_core/backends.py
index f5108fe3d..aa73b9281 100644
--- a/rootfs/api/backend.py
+++ b/rootfs/api/apps_extra/social_core/backends.py
@@ -4,7 +4,6 @@
 from django.contrib.auth import get_user_model
 from django.utils.translation import gettext_lazy
 
-from social_core.utils import cache as social_cache
 from social_core.backends.open_id_connect import OpenIdConnectAuth
 from rest_framework import exceptions
 
@@ -16,12 +15,6 @@
 class DryccOIDC(OpenIdConnectAuth):
     """Drycc Openid Connect authentication backend"""
     name = 'drycc'
-    AUTHORIZATION_URL = settings.SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL
-    ACCESS_TOKEN_URL = settings.SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL
-    USERINFO_URL = settings.SOCIAL_AUTH_DRYCC_USERINFO_URL
-    JWKS_URI = settings.SOCIAL_AUTH_DRYCC_JWKS_URI
-    OIDC_ENDPOINT = settings.SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT
-    DEFAULT_SCOPE = ['openid', 'profile', 'email']
     EXTRA_DATA = [
         ('id', 'id'),
         ('access_token', 'access_token'),
@@ -32,20 +25,10 @@ class DryccOIDC(OpenIdConnectAuth):
         ('scope', 'scope'),
     ]
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    @social_cache(ttl=86400)
-    def oidc_config(self):
-        return self.get_json(
-            self.OIDC_ENDPOINT + '/.well-known/openid-configuration/'
-        )
-
     def get_user_data(self, access_token):
         """Loads user data from service"""
-        url = settings.SOCIAL_AUTH_DRYCC_USERINFO_URL
         response = self.get_json(
-            url,
+            self.userinfo_url(),
             headers={
                 'authorization': 'Bearer ' + access_token
             },
@@ -62,12 +45,13 @@ def get_user_data(self, access_token):
         }
 
     def refresh_token(self, refresh_token):
+        # Get token URL from OIDC discovery if not already cached
         return self.get_json(
-            settings.SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL,
+            self.access_token_url(),
             method='POST',
             data={
                 'grant_type': 'refresh_token',
-                'client_id': settings.SOCIAL_AUTH_DRYCC_KEY,
+                'client_id': self.get_key_and_secret()[0],
                 'refresh_token': refresh_token,
             },
         )
diff --git a/rootfs/api/apps_extra/social_core/pipelines.py b/rootfs/api/apps_extra/social_core/pipelines.py
new file mode 100644
index 000000000..5eefe0624
--- /dev/null
+++ b/rootfs/api/apps_extra/social_core/pipelines.py
@@ -0,0 +1,6 @@
+from api.serializers import UserSerializer
+
+
+def update_or_create(backend, user, response, *args, **kwargs):
+    user, created = UserSerializer.update_or_create(response)
+    return {'is_new': created, 'user': user}
diff --git a/rootfs/api/authentication.py b/rootfs/api/authentication.py
index 5e40f0380..63559903e 100644
--- a/rootfs/api/authentication.py
+++ b/rootfs/api/authentication.py
@@ -47,7 +47,7 @@ def authenticate(self, request):
             return None
         try:
             if token_type == 'bearer':  # drycc oauth access token
-                from api.backend import OauthCacheManager
+                from api.apps_extra.social_core.backends import OauthCacheManager
                 return OauthCacheManager().get_user(token), token
             # drycc token
             user = cache.get(token, None)
@@ -69,7 +69,7 @@ def authenticate_credentials(self, key):
             raise exceptions.AuthenticationFailed(gettext_lazy('User inactive or deleted.'))
         if token.expires():
             try:
-                from api.backend import OauthCacheManager
+                from api.apps_extra.social_core.backends import OauthCacheManager
                 user = OauthCacheManager().get_user(token.oauth['access_token'])
                 cache.set(key, user, timeout=token.oauth['expires_in'])
                 token.refresh_token()
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index acedee88d..67baa73c5 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -120,7 +120,7 @@ def generate_key(cls):
         ) + ''.join(random.choices(string.ascii_letters, k=96))
 
     def refresh_token(self):
-        from api.backend import DryccOIDC
+        from api.apps_extra.social_core.backends import DryccOIDC
         drycc_open_connect = DryccOIDC()
         self.oauth = drycc_open_connect.refresh_token(self.oauth['refresh_token'])
         self.save()
diff --git a/rootfs/api/pipeline.py b/rootfs/api/pipeline.py
deleted file mode 100644
index 0426ccb71..000000000
--- a/rootfs/api/pipeline.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from api.serializers import UserSerializer
-
-
-def update_or_create(backend, user, response, *args, **kwargs):
-    user, created = UserSerializer.update_or_create(response)
-
-    if not created:
-        return {'is_new': False}
-
-    return {
-        'is_new': True,
-        'user': user
-    }
-
-
-def load_extra_data(backend, details, response, uid, user, *args, **kwargs):
-    social = kwargs.get('social') or \
-             backend.strategy.storage.user.get_social_auth(backend.name, uid)
-    if social:
-        extra_data = backend.extra_data(user, uid, response, details,
-                                        *args, **kwargs)
-        social.set_extra_data(extra_data)
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index d570e8726..304bc416d 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -127,6 +127,7 @@ def randstr(k):
 AUTHENTICATION_BACKENDS = (
     "django.contrib.auth.backends.ModelBackend",
     "guardian.backends.ObjectPermissionBackend",
+    "api.apps_extra.social_core.backends.DryccOIDC",
 )
 GUARDIAN_GET_INIT_ANONYMOUS_USER = 'api.models.base.get_anonymous_user_instance'
 ANONYMOUS_USER_NAME = os.environ.get('ANONYMOUS_USER_NAME', 'AnonymousUser')
@@ -431,41 +432,19 @@ def randstr(k):
     f'{DRYCC_PASSPORT_URL}/user/login/done/',
 )
 
+# social auth settings
 SOCIAL_AUTH_DRYCC_KEY = os.environ.get(
     "DRYCC_PASSPORT_KEY",
     os.environ.get("SOCIAL_AUTH_DRYCC_KEY"),
 )
-
 SOCIAL_AUTH_DRYCC_SECRET = os.environ.get(
     'DRYCC_PASSPORT_SECRET',
     os.environ.get("SOCIAL_AUTH_DRYCC_SECRET"),
 )
-
-SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL',
-    f'{DRYCC_PASSPORT_URL}/oauth/authorize/',
-)
-SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL',
-    f'{DRYCC_PASSPORT_URL}/oauth/token/'
-)
-SOCIAL_AUTH_DRYCC_ACCESS_API_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_ACCESS_API_URL',
-    f'{DRYCC_PASSPORT_URL}'
-)
-SOCIAL_AUTH_DRYCC_USERINFO_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_USERINFO_URL',
-    f'{DRYCC_PASSPORT_URL}/oauth/userinfo/'
-)
-SOCIAL_AUTH_DRYCC_JWKS_URI = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_JWKS_URI',
-    f'{DRYCC_PASSPORT_URL}/oauth/.well-known/jwks.json'
-)
 SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT = os.environ.get(
     'SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT',
     f'{DRYCC_PASSPORT_URL}/oauth'
 )
-
 SOCIAL_AUTH_JSONFIELD_ENABLED = True
 SOCIAL_AUTH_PIPELINE = (
     'social_core.pipeline.social_auth.social_details',
@@ -473,12 +452,11 @@ def randstr(k):
     'social_core.pipeline.social_auth.social_user',
     'social_core.pipeline.user.get_username',
     'social_core.pipeline.social_auth.associate_by_email',
-    'api.pipeline.update_or_create',
+    'api.apps_extra.social_core.pipelines.update_or_create',
     'social_core.pipeline.social_auth.associate_user',
     'social_core.pipeline.social_auth.load_extra_data',
     'social_core.pipeline.user.user_details',
 )
-AUTHENTICATION_BACKENDS = ("api.backend.DryccOIDC", ) + AUTHENTICATION_BACKENDS
 DRYCC_CACHE_USER_TIME = int(os.environ.get('DRYCC_CACHE_USER_TIME', 30 * 60))
 
 # Cache Valkey Configuration
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index 37126dc23..ef30259e9 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -47,7 +47,7 @@
 from social_django.views import _do_login
 from social_core.utils import setting_name
 from api import admissions, utils
-from api.backend import OauthCacheManager
+from api.apps_extra.social_core.backends import OauthCacheManager
 from api.apps_extra.social_core.actions import do_auth, do_complete
 
 
@@ -122,12 +122,15 @@ def _create_browser_response(self, key):
         return redirect(f"{uri[0:uri.find(self.request.path)]}/v2/login/drycc/?key={key}")
 
     def _create_interactive_response(self, username, password, key):
+        # Get token endpoint from OIDC discovery
+        token_url = oauth_cache_manager.drycc_oauth.access_token_url()
+        client_id, client_secret = oauth_cache_manager.drycc_oauth.get_key_and_secret()
         response = requests.post(
-            settings.SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL,
+            token_url,
             data={
                 'grant_type': 'password',
-                'client_id': settings.SOCIAL_AUTH_DRYCC_KEY,
-                'client_secret': settings.SOCIAL_AUTH_DRYCC_SECRET,
+                'client_id': client_id,
+                'client_secret': client_secret,
                 'username': username,
                 'password': password,
             },

From 8c2772677453816e85b07d61e11f2c22519fcec1 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 18 Jan 2026 19:08:08 +0800
Subject: [PATCH 198/200] chore(deps): bump pyasn1 in /rootfs in the pip group
 across 1 directory (#202)

Bumps the pip group with 1 update in the /rootfs directory: [pyasn1](https://github.com/pyasn1/pyasn1).


Updates `pyasn1` from 0.6.1 to 0.6.2
- [Release notes](https://github.com/pyasn1/pyasn1/releases)
- [Changelog](https://github.com/pyasn1/pyasn1/blob/main/CHANGES.rst)
- [Commits](https://github.com/pyasn1/pyasn1/compare/v0.6.1...v0.6.2)

---
updated-dependencies:
- dependency-name: pyasn1
  dependency-version: 0.6.2
  dependency-type: direct:production
  dependency-group: pip
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 rootfs/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 2a250d3d8..c8ca06e04 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -15,7 +15,7 @@ idna==3.11
 jsonschema==4.25.1
 morph==0.1.5
 packaging==25.0
-pyasn1==0.6.1
+pyasn1==0.6.2
 psycopg[binary]==3.2.12
 pyOpenSSL==25.3.0
 ndg-httpsclient==0.5.1

From abe3fa3509b99474e29b39612743926b42b5dc38 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Thu, 12 Mar 2026 13:52:22 +0800
Subject: [PATCH 199/200] chore(charts): add personal to reserved name

---
 charts/controller/files/reserved-name-patterns.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/charts/controller/files/reserved-name-patterns.txt b/charts/controller/files/reserved-name-patterns.txt
index 71574be55..1b6ca23bc 100644
--- a/charts/controller/files/reserved-name-patterns.txt
+++ b/charts/controller/files/reserved-name-patterns.txt
@@ -9,4 +9,5 @@
 ^metallb$
 ^mount-s3$
 ^topolvm$
-^rook-ceph$
\ No newline at end of file
+^rook-ceph$
+^personal$
\ No newline at end of file

From 01fa018f490e82a20012f3478cd3ebb618cc72a3 Mon Sep 17 00:00:00 2001
From: duanhongyi <duanhongyi@doopai.com>
Date: Tue, 17 Mar 2026 08:28:48 +0800
Subject: [PATCH 200/200] feat(workspace): migrate controller from owner-based
 auth to workspace model

---
 rootfs/api/admin.py                           |  14 -
 rootfs/api/consumers.py                       |  11 +-
 rootfs/api/exceptions.py                      |   2 +
 .../commands/upload_network_usage.py          |   2 +-
 .../commands/upload_volume_usage.py           |   2 +-
 rootfs/api/manager.py                         |  16 +-
 ..._migration_permissions_and_certificates.py |  12 -
 ...owner_remove_appsettings_owner_and_more.py | 119 ++++
 ...key_api_key_fingerp_4e77c5_idx_and_more.py |  25 +
 rootfs/api/models/app.py                      |  74 +--
 rootfs/api/models/appsettings.py              |  11 +-
 rootfs/api/models/base.py                     |   9 +-
 rootfs/api/models/blocklist.py                |  14 +-
 rootfs/api/models/build.py                    |   8 +-
 rootfs/api/models/certificate.py              | 123 ++---
 rootfs/api/models/config.py                   |   1 -
 rootfs/api/models/domain.py                   |   1 -
 rootfs/api/models/gateway.py                  |   6 +-
 rootfs/api/models/key.py                      |   3 +
 rootfs/api/models/limit.py                    |   3 +
 rootfs/api/models/release.py                  |  25 +-
 rootfs/api/models/resource.py                 |   3 +-
 rootfs/api/models/service.py                  |   1 -
 rootfs/api/models/tls.py                      |   5 +-
 rootfs/api/models/volume.py                   |   3 +-
 rootfs/api/models/workspace.py                | 108 ++++
 rootfs/api/permissions.py                     |  98 +---
 rootfs/api/serializers/__init__.py            |  65 ++-
 rootfs/api/settings/celery.py                 |   4 -
 rootfs/api/settings/production.py             |  13 +-
 rootfs/api/signals.py                         |  11 +-
 rootfs/api/tasks.py                           |  24 +-
 .../workspace/workspace_invitation.html       |   7 +
 .../workspace_invitation_accept.html          | 107 ++++
 rootfs/api/tests/__init__.py                  |  49 +-
 rootfs/api/tests/test_app.py                  | 159 +++---
 rootfs/api/tests/test_build.py                |   3 +-
 rootfs/api/tests/test_certificate.py          | 104 +++-
 .../api/tests/test_certificate_use_case_1.py  |   6 +-
 .../api/tests/test_certificate_use_case_2.py  |   8 +-
 .../api/tests/test_certificate_use_case_3.py  |   8 +-
 .../api/tests/test_certificate_use_case_4.py  |  10 +-
 .../api/tests/test_certificate_use_case_5.py  |  10 +-
 rootfs/api/tests/test_config.py               |  27 +-
 rootfs/api/tests/test_consumers.py            |   1 -
 rootfs/api/tests/test_domain.py               |   3 +-
 rootfs/api/tests/test_event.py                |   6 +-
 rootfs/api/tests/test_gateway.py              |  12 +-
 rootfs/api/tests/test_hooks.py                |  10 -
 rootfs/api/tests/test_perm.py                 | 406 +++++---------
 rootfs/api/tests/test_pods.py                 |  18 +-
 rootfs/api/tests/test_ptypes.py               |   6 +-
 rootfs/api/tests/test_release.py              |  12 +-
 rootfs/api/tests/test_resource.py             |   3 +-
 rootfs/api/tests/test_users.py                |  47 --
 rootfs/api/tests/test_volume.py               |   4 +-
 rootfs/api/tests/test_workflow_manager.py     |   6 +-
 rootfs/api/tests/test_workspace.py            | 196 +++++++
 rootfs/api/urls.py                            |  63 ++-
 rootfs/api/utils.py                           |   5 +
 rootfs/api/views.py                           | 520 ++++++++++--------
 rootfs/api/viewsets.py                        |  34 +-
 rootfs/requirements.txt                       |   6 +-
 63 files changed, 1568 insertions(+), 1104 deletions(-)
 create mode 100644 rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py
 create mode 100644 rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py
 create mode 100644 rootfs/api/models/workspace.py
 create mode 100644 rootfs/api/templates/workspace/workspace_invitation.html
 create mode 100644 rootfs/api/templates/workspace/workspace_invitation_accept.html
 delete mode 100644 rootfs/api/tests/test_users.py
 create mode 100644 rootfs/api/tests/test_workspace.py

diff --git a/rootfs/api/admin.py b/rootfs/api/admin.py
index c3710a944..199161af6 100644
--- a/rootfs/api/admin.py
+++ b/rootfs/api/admin.py
@@ -6,9 +6,7 @@
 
 
 from django.contrib import admin
-from guardian.admin import GuardedModelAdmin
 
-from .models import App
 from .models import Build
 from .models import Config
 from .models import Domain
@@ -16,18 +14,6 @@
 from .models import Release
 
 
-class AppAdmin(GuardedModelAdmin):
-    """Set presentation options for :class:`~api.models.App` models
-    in the Django admin.
-    """
-    date_hierarchy = 'created'
-    list_display = ('id', 'owner')
-    list_filter = ('owner',)
-
-
-admin.site.register(App, AppAdmin)
-
-
 class BuildAdmin(admin.ModelAdmin):
     """Set presentation options for :class:`~api.models.Build` models
     in the Django admin.
diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index e3995d771..47d680a83 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -4,6 +4,7 @@
 import ssl
 import aiohttp
 import asyncio
+from collections import namedtuple
 from urllib.parse import urljoin
 from django.conf import settings
 from django.core.cache import cache
@@ -22,11 +23,12 @@
 
 from .models.app import App
 from .models.volume import Volume
-from .permissions import has_app_permission
+from .permissions import IsAppUser, get_app_status
 
 
 class AppPermChecker(object):
     timeout = 60 * 60
+    check_permission = IsAppUser().has_object_permission
 
     def __init__(self, scope):
         self.scope = scope
@@ -40,8 +42,11 @@ async def has_perm(self):
         if permission is None:
             try:
                 app = await App.objects.aget(id=app_id)
-                permission = await sync_to_async(has_app_permission)(
-                    self.scope["user"], app, "GET")
+                request = namedtuple("Request", ["user", "method"])(self.scope["user"], "GET")
+                if await sync_to_async(self.check_permission)(request, None, app):
+                    permission = await sync_to_async(get_app_status)(app)
+                else:
+                    permission = (False, "permission denied")
                 if permission[0]:
                     await cache.aset(key, permission, timeout=self.timeout)
             except App.DoesNotExist:
diff --git a/rootfs/api/exceptions.py b/rootfs/api/exceptions.py
index bf6ce99c5..38db7b9b9 100644
--- a/rootfs/api/exceptions.py
+++ b/rootfs/api/exceptions.py
@@ -42,6 +42,8 @@ def custom_exception_handler(exc, context):
     # No response means DRF couldn't handle it
     # Output a generic 500 in a JSON format
     if response is None:
+        import traceback
+        traceback.print_exc()
         logging.exception('Uncaught Exception', exc_info=exc)
         set_rollback()
         return Response({'detail': 'Server Error'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/rootfs/api/management/commands/upload_network_usage.py b/rootfs/api/management/commands/upload_network_usage.py
index c24465655..fd80ea387 100644
--- a/rootfs/api/management/commands/upload_network_usage.py
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -27,7 +27,7 @@ def _upload_network_usage(self, start_time, app_map, timestamp):
             pod_name = metric['pod']
             networks.append({
                 "app_id":  str(app_map[metric['namespace']].uuid),
-                "owner": app_map[metric['namespace']].owner_id,
+                "workspace": app_map[metric['namespace']].workspace_id,
                 "type": "network",
                 "unit": "bytes",
                 "name": metric['direction'],
diff --git a/rootfs/api/management/commands/upload_volume_usage.py b/rootfs/api/management/commands/upload_volume_usage.py
index 1d4b80e57..9852e42ff 100644
--- a/rootfs/api/management/commands/upload_volume_usage.py
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -27,7 +27,7 @@ def _upload_volume_usage(self, start_time, app_map, timestamp):
             pvc_name = metric['persistentvolumeclaim']
             volumes.append({
                 "app_id":  str(app_map[metric['namespace']].uuid),
-                "owner": app_map[metric['namespace']].owner_id,
+                "workspace": app_map[metric['namespace']].workspace_id,
                 "type": "volume",
                 "unit": "bytes",
                 "name": metric["storageclass"],
diff --git a/rootfs/api/manager.py b/rootfs/api/manager.py
index a7a178db2..53b22eb7f 100644
--- a/rootfs/api/manager.py
+++ b/rootfs/api/manager.py
@@ -54,23 +54,23 @@ def delete(self, url, **kwargs):
         return self.request('delete', url, **kwargs)
 
 
-class UserAPI(ManagerAPI):
+class WorkspaceAPI(ManagerAPI):
 
-    def get_status(self, id):
+    def get_status(self, workspace_id):
         """
         {
             "is_active": False,
             "message": "The user is in arrears"
         }
         """
-        key = f"user:status:{id}"
+        key = f"workspace:status:{workspace_id}"
         status = cache.get(key)
         if not status:
-            url = f"{settings.WORKFLOW_MANAGER_URL}/users/{id}/status/"
+            url = f"{settings.WORKFLOW_MANAGER_URL}/workspaces/{workspace_id}/status/"
             try:
                 status = self.get(url=url, timeout=self.timeout).json()
             except requests.exceptions.Timeout as ex:
-                msg = f"request user {id} timeout, skipping verification."
+                msg = f"request workspace {workspace_id} timeout, skipping verification."
                 status = {"is_active": True, "message": msg}
                 logger.error(msg)
                 logger.exception(ex)
@@ -78,6 +78,10 @@ def get_status(self, id):
         return status
 
 
+class UserAPI(WorkspaceAPI):
+    """Backward-compatible alias for legacy call sites."""
+
+
 class UsageAPI(ManagerAPI):
 
     def post(self, usages: List[Dict[str, str]]):
@@ -85,7 +89,7 @@ def post(self, usages: List[Dict[str, str]]):
         [
             {
                 "app_id":  "test",
-                "owner": "test",
+                "workspace": "test",
                 "name": "web",
                 "type": "limits",
                 "unit": "std1.large.c1m1",
diff --git a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
index f8704081b..63e87c4ab 100644
--- a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -1,21 +1,10 @@
 # Generated by Django 4.2.15 on 2024-09-03 03:48
 
 from django.db import migrations
-from guardian.shortcuts import assign_perm, get_users_with_perms, remove_perm
-from api.models.app import VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
 from api.models.domain import Domain
 from api.models.certificate import Certificate
 
 
-def migration_permission(apps, schema_editor):
-    App = apps.get_model('api', 'App')
-    for app in App.objects.all():
-        for user in get_users_with_perms(app):
-            remove_perm('use_app', user, app)
-            assign_perm(VIEW_APP_PERMISSION.codename, user, app)
-            assign_perm(CHANGE_APP_PERMISSION.codename, user, app)
-
-
 def migration_certificate(apps, schema_editor):
     for domain in Domain.objects.all():
         if domain.certificate:
@@ -38,6 +27,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(migration_permission),
         migrations.RunPython(migration_certificate),
     ]
diff --git a/rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py b/rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py
new file mode 100644
index 000000000..5dfb1ee61
--- /dev/null
+++ b/rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py
@@ -0,0 +1,119 @@
+# Generated by Django 5.2.9 on 2026-03-23 05:42
+
+import api.models.workspace
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0027_rename_lifecycle_post_start_config_lifecycle_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Workspace',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.SlugField(max_length=150, unique=True, validators=[api.models.workspace.validate_workspace_name], verbose_name='workspace name')),
+                ('email', models.EmailField(max_length=254, verbose_name='email address')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='app',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='appsettings',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='build',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='certificate',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='config',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='domain',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='gateway',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='release',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='route',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='resource',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='service',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='tls',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='volume',
+            name='owner',
+        ),
+        migrations.AlterField(
+            model_name='blocklist',
+            name='type',
+            field=models.PositiveIntegerField(choices=[(1, 'app'), (2, 'workspace')]),
+        ),
+        migrations.AddField(
+            model_name='app',
+            name='workspace',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='api.workspace'),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name='WorkspaceInvitation',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('email', models.EmailField(max_length=254, verbose_name='email address')),
+                ('token', models.CharField(max_length=128, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('accepted', models.BooleanField(default=False, verbose_name='accepted')),
+                ('inviter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.workspace')),
+            ],
+            options={
+                'unique_together': {('email', 'workspace')},
+            },
+        ),
+        migrations.CreateModel(
+            name='WorkspaceMember',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('role', models.CharField(choices=[('admin', 'Admin'), ('member', 'Member'), ('viewer', 'Viewer')], max_length=50)),
+                ('alerts', models.BooleanField(default=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.workspace')),
+            ],
+            options={
+                'unique_together': {('user', 'workspace')},
+            },
+        ),
+    ]
diff --git a/rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py b/rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py
new file mode 100644
index 000000000..454a81db8
--- /dev/null
+++ b/rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py
@@ -0,0 +1,25 @@
+# Generated by Django 5.2.9 on 2026-03-26 08:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0028_workspace_remove_app_owner_remove_appsettings_owner_and_more'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='key',
+            index=models.Index(fields=['fingerprint'], name='api_key_fingerp_4e77c5_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='limitplan',
+            index=models.Index(fields=['spec', 'cpu', 'memory'], name='api_limitpl_spec_id_f95ef2_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='release',
+            index=models.Index(fields=['app', 'failed', 'state'], name='api_release_app_id_9849f2_idx'),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index 8da9a0d8b..3e7914035 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -10,7 +10,7 @@
 import socket
 from contextlib import closing
 from urllib.parse import urljoin
-from collections import OrderedDict, namedtuple
+from collections import OrderedDict
 from datetime import datetime, timezone
 
 from docker import auth as docker_auth
@@ -38,47 +38,6 @@
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
-AppPermission = namedtuple('AppPermission', ['shortname', 'codename', 'description'])
-VIEW_APP_PERMISSION = AppPermission("view", "view_app", "can view app")
-CHANGE_APP_PERMISSION = AppPermission("change", "change_app", "can change app")
-DELETE_APP_PERMISSION = AppPermission("delete", "delete_app", "can delete app")
-
-
-class AppPermissionRegistry(object):
-
-    def __init__(self):
-        self.tags = {}
-        self.permissions = set()
-
-    def get(self, q):
-        permissions = [
-            permission for permission in self.permissions
-            if q == permission.shortname or q == permission.codename
-        ]
-        permissions.extend([
-            permission for permission in self.permissions
-            if permission in self.tags and q in self.tags[permission]
-        ])
-        return permissions[0] if permissions else None
-
-    def register(self, permission, tags=None):
-        if tags:
-            self.tags[permission] = tags
-        self.permissions.add(permission)
-
-    @property
-    def codenames(self):
-        return [permission[1] for permission in self.permissions]
-
-    @property
-    def shortnames(self):
-        return [permission[0] for permission in self.permissions]
-
-
-app_permission_registry = AppPermissionRegistry()
-app_permission_registry.register(VIEW_APP_PERMISSION, ["GET", "HEAD", "OPTION"])
-app_permission_registry.register(CHANGE_APP_PERMISSION, ["POST", "PUT", "PATCH"])
-app_permission_registry.register(DELETE_APP_PERMISSION, ["DELETE"])
 
 
 # http://kubernetes.io/v1.1/docs/design/identifiers.html
@@ -109,9 +68,9 @@ class App(UuidAuditedModel):
     Application used to service requests on behalf of end-users
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     id = models.SlugField(max_length=63, unique=True, null=True,
                           validators=[validate_app_id])
+    workspace = models.ForeignKey('Workspace', on_delete=models.CASCADE)
     structure = models.JSONField(
         default=dict, blank=True, validators=[validate_app_structure])
     suspended_state = models.JSONField(default=dict, blank=True)
@@ -150,6 +109,12 @@ def save(self, *args, **kwargs):
     def lock(self):
         return CacheLock(f"app:lock:{self.id}")
 
+    def _default_actor(self):
+        member = self.workspace.workspacemember_set.select_related('user').first()
+        if member:
+            return member.user
+        raise DryccException("workspace has no members")
+
     @property
     def ptypes(self):
         return list(self.structure.keys())
@@ -162,7 +127,8 @@ def scheduler(self):
         directly reference using ID instead.
         """
         scheduler = super(App, self).scheduler
-        scheduler.metadata["annotations"]["drycc.cc/project_id"] = str(self.id)
+        scheduler.metadata["annotations"]["drycc.cc/app_id"] = str(self.id)
+        scheduler.metadata["annotations"]["drycc.cc/workspace_id"] = str(self.workspace_id)
         return scheduler
 
     def check_ptypes(self, ptypes: set):
@@ -201,7 +167,7 @@ def create(self, *args, **kwargs):  # noqa
             self.release_set.latest()
         except Release.DoesNotExist:
             Release.objects.create(
-                version=1, owner=self.owner, app=self,
+                version=1, app=self,
                 config=cfg, build=None
             )
 
@@ -220,11 +186,11 @@ def create(self, *args, **kwargs):  # noqa
             self.appsettings_set.latest()
         except AppSettings.DoesNotExist:
             AppSettings.objects.create(
-                owner=self.owner, app=self, routable=True, autodeploy=True, autorollback=True)
+                app=self, routable=True, autodeploy=True, autorollback=True)
         try:
             self.tls_set.latest()
         except TLS.DoesNotExist:
-            TLS.objects.create(owner=self.owner, app=self)
+            TLS.objects.create(app=self)
 
     def delete(self, *args, **kwargs):
         """Delete this application including all containers"""
@@ -297,7 +263,7 @@ def pipeline(self, release, ptypes, force_deploy=False):
                     for run in release.get_runners(ptypes):
                         self.log(f"{prefix} starts running pipeline.run {run['image']}")
                         job_name = self.run(
-                            self.owner, run['image'], command=run['command'],
+                            self._default_actor(), run['image'], command=run['command'],
                             args=run['args'], timeout=run['timeout'], expires=run['timeout'],
                             envs=self._build_env_vars(release, run['ptype']),
                         )
@@ -374,7 +340,7 @@ def clean(self, release=None, ptypes=None):
                 if ptype not in release.ptypes and self.structure.get(ptype, 0) > 0:
                     if ptypes is None or ptype in ptypes:
                         removed[ptype] = 0
-            self.scale(self.owner, removed)
+            self.scale(self._default_actor(), removed)
         self._merge_structure(release, pre_release)
         # clean k8s resources, ptype not in structure
         labels = {'heritage': 'drycc'}
@@ -732,7 +698,7 @@ def to_usages(self, timestamp: float):
             plan = config.limits.get(ptype)
             usage.append({
                 "app_id": str(self.uuid),
-                "owner": self.owner_id,
+                "workspace": self.workspace_id,
                 "name": plan,
                 "type": "limits",
                 "unit": "number",
@@ -936,7 +902,7 @@ def _set_default_config(self):
             limits[PTYPE_WEB] = config.limits.get(PTYPE_WEB, plan.id)
             limits[PTYPE_RUN] = config.limits.get(PTYPE_RUN, plan.id)
         except Config.DoesNotExist:
-            config = Config.objects.create(owner=self.owner, app=self, limits=limits)
+            config = Config.objects.create(app=self, limits=limits)
         for ptype in self.ptypes:
             limits[ptype] = config.limits.get(ptype, plan.id)
         if limits != config.limits:
@@ -949,7 +915,7 @@ def _create_default_ingress(self, target_port):
         try:
             service = self.service_set.filter(ptype=PTYPE_WEB).latest()
         except Service.DoesNotExist:
-            service = Service(owner=self.owner, app=self, ptype=PTYPE_WEB)
+            service = Service(app=self, ptype=PTYPE_WEB)
             service.add_port(DEFAULT_HTTP_PORT, "TCP", target_port)
             service.save()
         else:
@@ -961,7 +927,7 @@ def _create_default_ingress(self, target_port):
             if gateway.change_default_tls():
                 gateway.save()
         except Gateway.DoesNotExist:
-            gateway = Gateway(app=self, owner=self.owner, name=self.id)
+            gateway = Gateway(app=self, name=self.id)
             added, msg = gateway.add(DEFAULT_HTTP_PORT, "HTTP")
             if not added:
                 raise DryccException(msg)
@@ -972,7 +938,7 @@ def _create_default_ingress(self, target_port):
             if route.change_default_tls():
                 route.save()
         except Route.DoesNotExist:
-            route = Route(app=self, owner=self.owner, kind="HTTPRoute", name=self.id,
+            route = Route(app=self, kind="HTTPRoute", name=self.id,
                           rules=[{"backendRefs": [{"kind": "Service", "name": service.name,
                                                    "port": DEFAULT_HTTP_PORT, "weight": 100}]}])
             attached, msg = route.attach(gateway.name, DEFAULT_HTTP_PORT)
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index 4f9066afe..32c1da8f7 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -20,7 +20,6 @@ class AppSettings(UuidAuditedModel):
     Instance of Application settings used by scheduler
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     routable = models.BooleanField(default=None)
     autodeploy = models.BooleanField(default=None)
@@ -77,7 +76,9 @@ def _update_field(self, field, previous_settings):
         elif new is None and isinstance(self._meta.get_field(field), models.BooleanField):
             setattr(self, field, True)
         elif old != new:
-            self.summary += ["{} changed {} from {} to {}".format(self.owner, field, old, new)]
+            self.summary += [
+                "{} changed {} from {} to {}".format(self.app.workspace.name, field, old, new)
+            ]
 
     def _update_autoscale(self, previous_settings):
         data = getattr(previous_settings, 'autoscale', {}).copy()
@@ -117,7 +118,7 @@ def _update_autoscale(self, previous_settings):
             deleted = 'deleted autoscale for process type ' + deleted if deleted else ''
             changes = ', '.join(i for i in (added, changed, deleted) if i)
             if changes:
-                self.summary += ["{} {}".format(self.owner, changes)]
+                self.summary += ["{} {}".format(self.app.workspace.name, changes)]
 
     def _update_label(self, previous_settings):
         data = getattr(previous_settings, 'label', {}).copy()
@@ -149,7 +150,7 @@ def _update_label(self, previous_settings):
             if changes:
                 if self.summary:
                     self.summary += ' and '
-                self.summary += ["{} {}".format(self.owner, changes)]
+                self.summary += ["{} {}".format(self.app.workspace.name, changes)]
 
     @transaction.atomic
     def save(self, ignore_update_fields=None, *args, **kwargs):
@@ -175,7 +176,7 @@ def save(self, ignore_update_fields=None, *args, **kwargs):
 
         if not self.summary and previous_settings:
             self.delete()
-            raise AlreadyExists("{} changed nothing".format(self.owner))
+            raise AlreadyExists("{} changed nothing".format(self.app.workspace.name))
         super(AppSettings, self).save(**kwargs)
         summary = ' '.join(self.summary)
         self.log('summary of app setting changes: {}'.format(summary), logging.DEBUG)
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index 67baa73c5..f3310d3dc 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -5,7 +5,6 @@
 from datetime import timedelta
 from functools import partial
 from django.db import models
-from django.conf import settings
 from django.utils import timezone
 from django.contrib.auth.models import AbstractUser
 from django.utils.translation import gettext_lazy as _
@@ -35,9 +34,6 @@
 PTYPE_MAX_LENGTH = 63
 
 
-def get_anonymous_user_instance(user): return user(id=-1, username=settings.ANONYMOUS_USER_NAME)
-
-
 class AuditedModel(models.Model):
     """Add created and updated fields to a model."""
 
@@ -66,9 +62,8 @@ def app_log(self, app_id, msg):
     def scheduler(self):
         annotations = {}
         if hasattr(self, 'app'):
-            annotations["drycc.cc/project_id"] = str(self.app.id)
-        if hasattr(self, 'owner'):
-            annotations["drycc.cc/account_id"] = str(self.owner.id)
+            annotations["drycc.cc/app_id"] = str(self.app.id)
+            annotations["drycc.cc/workspace_id"] = str(self.app.workspace_id)
         return get_scheduler(metadata={"annotations": annotations})
 
 
diff --git a/rootfs/api/models/blocklist.py b/rootfs/api/models/blocklist.py
index 306aa4d09..e51a2b613 100644
--- a/rootfs/api/models/blocklist.py
+++ b/rootfs/api/models/blocklist.py
@@ -1,7 +1,6 @@
 import logging
 import functools
 from django.db import models
-from django.contrib.auth import get_user_model
 from scheduler import KubeHTTPException
 from api.exceptions import ServiceUnavailable
 from api.utils import apply_tasks
@@ -9,17 +8,15 @@
 from .base import UuidAuditedModel
 
 
-User = get_user_model()
 logger = logging.getLogger(__name__)
 
 
 class Blocklist(UuidAuditedModel):
     """
-    You can block apps or users.
-    If a user is blocked, all apps owned by the user will be stopped.
-    The apps managed by the user will not be affected.
+    You can block apps or workspaces.
+    If a workspace is blocked, all apps under that workspace will be stopped.
     """
-    type_choices = [(1, "app", ), (2, "user")]
+    type_choices = [(1, "app", ), (2, "workspace")]
     id = models.CharField(max_length=128, db_index=True)
     type = models.PositiveIntegerField(choices=type_choices)
     remark = models.TextField(blank=True, null=True, default="Blocked for unknown reason")
@@ -27,8 +24,7 @@ class Blocklist(UuidAuditedModel):
     @property
     def related_apps(self):
         if self.type == 2:
-            user = User.objects.get(id=self.id)
-            return App.objects.filter(owner=user)
+            return App.objects.filter(workspace_id=self.id)
         else:
             return App.objects.filter(id=self.id)
 
@@ -42,7 +38,7 @@ def get_type(cls, name: str):
     @classmethod
     def get_blocklist(cls, app: App):
         return cls.objects.filter(
-            models.Q(id=app.id, type=1) | models.Q(id=app.owner_id, type=2)
+            models.Q(id=app.id, type=1) | models.Q(id=app.workspace_id, type=2)
         ).first()
 
     class Meta:
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 2b91d755c..96bee571b 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -17,7 +17,6 @@ class Build(UuidAuditedModel):
     Instance of a software build used by runtime nodes
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     image = models.TextField()
     stack = models.CharField(max_length=32)
@@ -79,7 +78,7 @@ def merge(self, config):
                 pipeline['env'] = ptype_env[pipeline['ptype']]
                 pipeline['config'] = config.values_refs.get(pipeline['ptype'], [])
             return Build.objects.create(
-                owner=config.owner, app=self.app, image=self.image, stack=self.stack, sha=self.sha,
+                app=self.app, image=self.image, stack=self.stack, sha=self.sha,
                 procfile=self.procfile, dryccfile=dryccfile, dockerfile=self.dockerfile,
             )
         return self
@@ -125,7 +124,7 @@ def create_release(self, user, *args, **kwargs):
                 if new_release.summary:
                     new_release.summary += " "
                 new_release.summary += "{} deployed {} which failed".format(
-                    self.owner, str(self.uuid)[:7])
+                    user, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
                 # avoid overwriting other fields
@@ -157,7 +156,6 @@ def _get_or_create_config(self):
                         values_ref[pipeline['ptype']] = [config_ref]
                     else:
                         values_ref[pipeline['ptype']].append(config_ref)
-        config = Config(
-            owner=self.owner, app=self.app, values=values, values_refs=values_ref)
+        config = Config(app=self.app, values=values, values_refs=values_ref)
         config.save(ignore_update_fields=["values", "values_refs"])
         return config
diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index 43b862427..15778f0aa 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -1,10 +1,8 @@
 import logging
-from OpenSSL import crypto
-from pyasn1.codec.der import decoder as der_decoder
-from pyasn1.type import univ, constraint
-from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
-from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
-from datetime import datetime
+
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.x509.oid import ExtensionOID, NameOID
 from pytz import utc
 
 from django.shortcuts import get_object_or_404
@@ -22,59 +20,57 @@
 logger = logging.getLogger(__name__)
 
 
-# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
-class SubjectAltName(BaseSubjectAltName):
-    '''ASN.1 implementation for subjectAltNames support'''
+def get_subj_alt_name(peer_cert):
+    try:
+        san_extension = peer_cert.extensions.get_extension_for_oid(
+            ExtensionOID.SUBJECT_ALTERNATIVE_NAME
+        )
+    except x509.ExtensionNotFound:
+        return []
+
+    return san_extension.value.get_values_for_type(x509.DNSName)
+
 
-    # There is no limit to how many SAN certificates a certificate may have,
-    #   however this needs to have some limit so we'll set an arbitrarily high
-    #   limit.
-    sizeSpec = univ.SequenceOf.sizeSpec + \
-        constraint.ValueSizeConstraint(1, 1024)
+def public_keys_match(cert_public_key, private_key):
+    def _public_key_der(key):
+        """Serialize any public key to DER SubjectPublicKeyInfo bytes."""
+        return key.public_bytes(
+            serialization.Encoding.DER,
+            serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
 
+    return _public_key_der(cert_public_key) == _public_key_der(
+        private_key.public_key()
+    )
 
-# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
-def get_subj_alt_name(peer_cert):
-    # Search through extensions
-    dns_name = []
-    if not SUBJ_ALT_NAME_SUPPORT:
-        return dns_name
-
-    general_names = SubjectAltName()
-    for i in range(peer_cert.get_extension_count()):
-        ext = peer_cert.get_extension(i)
-        ext_name = ext.get_short_name()
-        if ext_name != b'subjectAltName':
-            continue
-
-        # PyOpenSSL returns extension data in ASN.1 encoded form
-        ext_dat = ext.get_data()
-        decoded_dat = der_decoder.decode(ext_dat,
-                                         asn1Spec=general_names)
-
-        for name in decoded_dat:
-            if not isinstance(name, SubjectAltName):
-                continue
-            for entry in range(len(name)):
-                component = name.getComponentByPosition(entry)
-                if component.getName() != 'dNSName':
-                    continue
-                dns_name.append(str(component.getComponent()))
-
-    return dns_name
+
+def get_certificate_not_valid_after(certificate):
+    if hasattr(certificate, 'not_valid_after_utc'):
+        return certificate.not_valid_after_utc
+
+    return certificate.not_valid_after.replace(tzinfo=utc)
+
+
+def get_certificate_not_valid_before(certificate):
+    if hasattr(certificate, 'not_valid_before_utc'):
+        return certificate.not_valid_before_utc
+
+    return certificate.not_valid_before.replace(tzinfo=utc)
 
 
 def validate_certificate(value):
     try:
-        return crypto.load_certificate(crypto.FILETYPE_PEM, value)
-    except crypto.Error as e:
+        certificate_bytes = value.encode('utf-8') if isinstance(value, str) else value
+        return x509.load_pem_x509_certificate(certificate_bytes)
+    except (TypeError, ValueError) as e:
         raise ValidationError('Could not load certificate: {}'.format(e))
 
 
 def validate_private_key(value):
     try:
-        return crypto.load_privatekey(crypto.FILETYPE_PEM, value)
-    except crypto.Error as e:
+        private_key_bytes = value.encode('utf-8') if isinstance(value, str) else value
+        return serialization.load_pem_private_key(private_key_bytes, password=None)
+    except (TypeError, ValueError) as e:
         raise ValidationError('Could not load private key: {}'.format(e))
 
 
@@ -87,12 +83,8 @@ def validate_cert_pair(certificate, private_key):
         # The certificate and key should already have been validated
         raise SuspiciousOperation(e)
 
-    if pkey.type() == crypto.TYPE_RSA:
-        # Compare modulus n, to the factors p and q
-        priv_numbers = pkey.to_cryptography_key().private_numbers()
-        pub_modulus = cert.get_pubkey().to_cryptography_key().public_numbers().n
-        if pub_modulus != (priv_numbers.p * priv_numbers.q):
-            raise ValidationError('Certificate and private key do not match!')
+    if not public_keys_match(cert.public_key(), pkey):
+        raise ValidationError('Certificate and private key do not match!')
 
     # Return tuple if everything went ok
     return (cert, pkey)
@@ -102,7 +94,6 @@ class Certificate(AuditedModel):
     """
     Public and private key pair used to secure application traffic at the router.
     """
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     name = models.CharField(max_length=253, validators=[validate_label])
     # there is no upper limit on the size of an x.509 certificate
@@ -144,34 +135,26 @@ def save(self, *args, **kwargs):
         certificate, _ = validate_cert_pair(self.certificate, self.key)
 
         if not self.common_name:
-            self.common_name = certificate.get_subject().CN
+            common_names = certificate.subject.get_attributes_for_oid(NameOID.COMMON_NAME)
+            self.common_name = common_names[0].value if common_names else None
 
         # Grab expire date of the certificate
         if not self.expires:
-            # https://pyopenssl.readthedocs.org/en/latest/api/crypto.html#OpenSSL.crypto.X509.get_notAfter
-            # Convert bytes to string
-            timestamp = certificate.get_notAfter().decode(encoding='UTF-8')
-            # convert openssl's expiry date format to Django's DateTimeField format
-            self.expires = datetime.strptime(timestamp, '%Y%m%d%H%M%SZ').replace(tzinfo=utc)
+            self.expires = get_certificate_not_valid_after(certificate)
 
         # Grab the start date of the certificate
         if not self.starts:
-            # https://pyopenssl.readthedocs.org/en/latest/api/crypto.html#OpenSSL.crypto.X509.get_notBefore
-            # Convert bytes to string
-            timestamp = certificate.get_notBefore().decode(encoding='UTF-8')
-            # convert openssl's starts date format to Django's DateTimeField format
-            self.starts = datetime.strptime(timestamp, '%Y%m%d%H%M%SZ').replace(tzinfo=utc)
+            self.starts = get_certificate_not_valid_before(certificate)
 
         # process issuers - separate each key/value with a slash
-        issuer = certificate.get_issuer().get_components()
-        self.issuer = '/' + '/'.join('%s=%s' % (key.decode(encoding='UTF-8'), value.decode(encoding='UTF-8')) for key, value in issuer)  # noqa
+        self.issuer = certificate.issuer.rfc4514_string()
 
         # process subject - separate each key/value with a slash
-        subject = certificate.get_subject().get_components()
-        self.subject = '/' + '/'.join('%s=%s' % (key.decode(encoding='UTF-8'), value.decode(encoding='UTF-8')) for key, value in subject)  # noqa
+        self.subject = certificate.subject.rfc4514_string()
 
         # public fingerprint of certificate
-        self.fingerprint = certificate.digest('sha256').decode()
+        fingerprint = certificate.fingerprint(hashes.SHA256()).hex().upper()
+        self.fingerprint = ':'.join(fingerprint[i:i + 2] for i in range(0, len(fingerprint), 2))
 
         # SubjectAltName from the certificate - return a list
         self.san = get_subj_alt_name(certificate)
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 649434b76..51bc0cf9f 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -20,7 +20,6 @@ class Config(UuidAuditedModel):
                     "termination_grace_period", "registry")
     allof_fields = ("values", ) + ptype_fields
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     values = models.JSONField(default=list, blank=True)
     values_refs = models.JSONField(default=dict, blank=True)
diff --git a/rootfs/api/models/domain.py b/rootfs/api/models/domain.py
index bb5e8db71..a1caff610 100644
--- a/rootfs/api/models/domain.py
+++ b/rootfs/api/models/domain.py
@@ -7,7 +7,6 @@
 
 
 class Domain(AuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     domain = models.TextField(
         blank=False, null=False, unique=True,
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 71dfef91b..c4e68c323 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -3,7 +3,6 @@
 import threading
 from django.db import models
 from django.conf import settings
-from django.contrib.auth import get_user_model
 from django.db.models import Q
 from django.http import Http404
 
@@ -15,7 +14,6 @@
 from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT
 
 
-User = get_user_model()
 logger = logging.getLogger(__name__)
 
 TLS_PROTOCOLS = ("HTTPS", "TLS")
@@ -24,7 +22,6 @@
 
 class Gateway(AuditedModel):
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     name = models.CharField(max_length=63, db_index=True, validators=[validate_label])
     ports = models.JSONField(default=list)
 
@@ -150,7 +147,7 @@ def delete(self, *args, **kwargs):
     def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
-            "owner": self.owner_id,
+            "workspace": self.app.workspace_id,
             "name": settings.DRYCC_APP_GATEWAY_CLASS,
             "type": "gateway",
             "unit": "number",
@@ -196,7 +193,6 @@ class Route(AuditedModel):
     }
 
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     kind = models.CharField(max_length=15, choices=[
         (key, '/'.join(value)) for key, value in PROTOCOLS_CHOICES.items()])
     name = models.CharField(max_length=63, db_index=True)
diff --git a/rootfs/api/models/key.py b/rootfs/api/models/key.py
index b94867e59..08e94cdf8 100644
--- a/rootfs/api/models/key.py
+++ b/rootfs/api/models/key.py
@@ -33,6 +33,9 @@ class Meta:
         verbose_name = 'SSH Key'
         unique_together = (('owner', 'fingerprint'))
         ordering = ['public']
+        indexes = [
+            models.Index(fields=['fingerprint']),
+        ]
 
     def __str__(self):
         return "{}...{}".format(self.public[:18], self.public[-31:])
diff --git a/rootfs/api/models/limit.py b/rootfs/api/models/limit.py
index 616c29268..a72eda1a4 100644
--- a/rootfs/api/models/limit.py
+++ b/rootfs/api/models/limit.py
@@ -124,6 +124,9 @@ class LimitPlan(AuditedModel):
     class Meta:
         get_latest_by = 'created'
         ordering = ['priority']
+        indexes = [
+            models.Index(fields=['spec', 'cpu', 'memory']),
+        ]
 
     def __str__(self):
         return self.name
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index 7cf8c750f..8a9f8c2f8 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -32,7 +32,6 @@ class Release(UuidAuditedModel):
         ("crashed", "Release pipeline runtime crashed"),
         ("succeed", "Release pipeline runtime succeed"),
     )
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     state = models.TextField(choices=STATE_CHOICES, default=STATE_CHOICES[0][0])
     version = models.PositiveIntegerField()
@@ -49,6 +48,9 @@ class Meta:
         get_latest_by = 'created'
         ordering = ['-created']
         unique_together = (('app', 'version'),)
+        indexes = [
+            models.Index(fields=['app', 'failed', 'state']),
+        ]
 
     def __str__(self):
         return "{0}-{1}".format(self.app.id, self.version_name)
@@ -143,8 +145,7 @@ def new(self, user, config, build, summary=None):
         new_version = self.app.release_set.latest().version + 1
         # create new release and auto-increment version
         return Release.objects.create(
-            owner=user, app=self.app, config=config,
-            build=build, version=new_version, summary=summary
+            app=self.app, config=config, build=build, version=new_version, summary=summary
         )
 
     def get_port(self, ptype):
@@ -263,7 +264,7 @@ def rollback(self, user, ptypes=None, version=None):
                 if new_release.summary:
                     new_release.summary += " "
                 new_release.summary += "{} performed roll back to a release that failed".format(
-                    self.owner)
+                    user)
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
                 # avoid overwriting other fields
@@ -424,12 +425,14 @@ def save(self, *args, **kwargs):  # noqa
             old_config = prev_release.config if prev_release else None
             # if the build changed, log it and who pushed it
             if self.version == 1:
-                self.summary += "{} created initial release".format(self.app.owner)
+                self.summary += "{} created initial release".format(self.app.workspace.name)
             elif self.build != old_build:
                 if self.build.sha:
-                    self.summary += "{} deployed {}".format(self.build.owner, self.build.sha[:7])
+                    self.summary += "{} deployed {}".format(
+                        self.app.workspace.name, self.build.sha[:7])
                 else:
-                    self.summary += "{} deployed {}".format(self.build.owner, self.build.image)
+                    self.summary += "{} deployed {}".format(
+                        self.app.workspace.name, self.build.image)
             elif self.config != old_config:
                 for field, diff in self.config.diff(old_config).items():
                     diff_list = []
@@ -437,11 +440,13 @@ def save(self, *args, **kwargs):  # noqa
                         diff_list.append(f'{diff_type} {field} {", ".join(values)}')
                     if diff_list:
                         changes = ', '.join(diff_list)
-                        self.summary += "{} {}".format(self.config.owner, changes)
+                        self.summary += "{} {}".format(self.app.workspace.name, changes)
             if not self.summary:
                 if self.version == 1:
-                    self.summary = "{} created the initial release".format(self.owner)
+                    self.summary = "{} created the initial release".format(self.app.workspace.name)
                 else:
                     # There were no changes to this release
-                    raise AlreadyExists("{} changed nothing - release stopped".format(self.owner))
+                    raise AlreadyExists(
+                        "{} changed nothing - release stopped".format(self.app.workspace.name)
+                    )
         super(Release, self).save(*args, **kwargs)
diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index 42cdb346f..63ef0b8c2 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -13,7 +13,6 @@
 
 
 class Resource(UuidAuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     name = models.CharField(max_length=63, validators=[validate_label])
     plan = models.CharField(max_length=128)
@@ -210,7 +209,7 @@ def retrieve(self, *args, **kwargs):
     def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
-            "owner": self.owner_id,
+            "workspace": self.app.workspace_id,
             "name": self.plan,
             "type": "resource",
             "unit": "number",
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index 1651d3110..6b1c2711f 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -29,7 +29,6 @@
 
 
 class Service(AuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     ports = models.JSONField(
         default=list, validators=[partial(validate_json, schema=service_ports_schema)])
diff --git a/rootfs/api/models/tls.py b/rootfs/api/models/tls.py
index 9eaa6a081..b6a5b9262 100644
--- a/rootfs/api/models/tls.py
+++ b/rootfs/api/models/tls.py
@@ -24,7 +24,6 @@ def default_issuer():
 
 
 class TLS(UuidAuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     issuer = models.JSONField(default=default_issuer)
     https_enforced = models.BooleanField(null=True)
@@ -134,12 +133,12 @@ def _check_previous_tls_settings(self):
             if self.https_enforced is not None:
                 if previous_tls_settings.https_enforced == self.https_enforced:
                     raise AlreadyExists(
-                        "{} changed nothing".format(self.owner))
+                        "{} changed nothing".format(self.app.workspace.name))
                 self.certs_auto_enabled = previous_tls_settings.certs_auto_enabled
             elif self.certs_auto_enabled is not None:
                 if previous_tls_settings.certs_auto_enabled == self.certs_auto_enabled:
                     raise AlreadyExists(
-                        "{} changed nothing".format(self.owner))
+                        "{} changed nothing".format(self.app.workspace.name))
                 self.https_enforced = previous_tls_settings.https_enforced
             previous_tls_settings.delete()
         except TLS.DoesNotExist:
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 09d45813c..01a41b54c 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -23,7 +23,6 @@ class Volume(UuidAuditedModel):
         ("nfs", "network file system"),
         ("oss", "object storage service file"),
     )
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     name = models.CharField(max_length=63, validators=[validate_label])
     size = models.CharField(default='0G', max_length=128)
@@ -75,7 +74,7 @@ def log(self, message, level=logging.INFO):
     def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
-            "owner": self.owner_id,
+            "workspace": self.app.workspace_id,
             "name": self.type,
             "type": "volume",
             "unit": "bytes",
diff --git a/rootfs/api/models/workspace.py b/rootfs/api/models/workspace.py
new file mode 100644
index 000000000..7e5b6a689
--- /dev/null
+++ b/rootfs/api/models/workspace.py
@@ -0,0 +1,108 @@
+import re
+import logging
+from django.db import models
+from django.core.mail import send_mail
+from django.utils.translation import gettext_lazy as _
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from django.template.loader import render_to_string
+
+from rest_framework.exceptions import ValidationError
+from api.utils import validate_reserved_names, get_local_host
+
+User = get_user_model()
+logger = logging.getLogger(__name__)
+
+
+def validate_workspace_name(value):
+    """
+    Check that the value follows the kubernetes name constraints
+    """
+    match = re.match(r'^[0-9a-z]{5,}$', value)
+    if not match:
+        raise ValidationError("App name must start with an alphabetic character, cannot end with a"
+                              + " hyphen and can only contain a-z (lowercase), 0-9 and hyphens.")
+    validate_reserved_names(value)
+
+
+class Workspace(models.Model):
+    name = models.SlugField(
+        _("workspace name"),
+        max_length=150,
+        unique=True,
+        validators=[validate_workspace_name],
+    )
+    email = models.EmailField(_("email address"))
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+
+    def has_member(self, user, role=None):
+        kwargs = {'user': user, 'workspace': self}
+        if role:
+            kwargs['role'] = role
+        return WorkspaceMember.objects.filter(**kwargs).exists()
+
+    def __str__(self):
+        return self.name
+
+
+class WorkspaceMember(models.Model):
+    role_choices = [
+        ('admin', 'Admin'),
+        ('member', 'Member'),
+        ('viewer', 'Viewer'),
+    ]
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    role = models.CharField(max_length=50, choices=role_choices)
+    alerts = models.BooleanField(default=True)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+    workspace = models.ForeignKey(Workspace, on_delete=models.CASCADE)
+
+    def __str__(self):
+        return f"{self.user.username} - {self.workspace.name} ({self.role})"
+
+    class Meta:
+        unique_together = ('user', 'workspace')
+
+
+class WorkspaceInvitation(models.Model):
+    email = models.EmailField(_("email address"))
+    token = models.CharField(max_length=128, unique=True)
+    inviter = models.ForeignKey(User, on_delete=models.CASCADE)
+    created = models.DateTimeField(auto_now_add=True)
+    accepted = models.BooleanField(_("accepted"), default=False)
+    workspace = models.ForeignKey(Workspace, on_delete=models.CASCADE)
+
+    class Meta:
+        unique_together = ('email', 'workspace')
+
+    def accept(self):
+        if not self.accepted:
+            self.accepted = True
+            self.save()
+        user = User.objects.filter(email=self.email).first()
+        if not user:
+            return
+        WorkspaceMember.objects.get_or_create(
+            user=user, workspace=self.workspace, defaults={'role': 'member'})
+
+    def send_email(self, request):
+        cache_key = f"invitation:email:{self.email}"
+        cache.add(cache_key, 0, timeout=settings.DRYCC_INVITATION_EMAIL_TIMEOUT)
+        count = cache.incr(cache_key)
+        if count > settings.DRYCC_INVITATION_EMAIL_LIMIT:
+            raise ValidationError("Too many invitation emails, please try again later")
+        domain = get_local_host(request)
+        mail_subject = f'We Invite You to Join the {self.workspace.name} Workspace.'
+        message = render_to_string(
+            'workspace/workspace_invitation.html',
+            {'domain': domain, 'invitation': self}
+        )
+        send_mail(mail_subject, message, None, [self.email], fail_silently=True)
+
+    def __str__(self):
+        return f"Invitation for {self.email} to join {self.workspace.name}"
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index 208df5baf..4bd7f90d1 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -1,10 +1,9 @@
 import base64
 from rest_framework import permissions
 from django.conf import settings
-from django.contrib.auth.models import AnonymousUser
 from api import manager
-from api.models import app
 from api.models import blocklist
+from api.models.workspace import Workspace, WorkspaceMember
 
 
 def get_app_status(app):
@@ -12,42 +11,12 @@ def get_app_status(app):
     if block:
         return False, block.remark
     if settings.WORKFLOW_MANAGER_URL:
-        status = manager.UserAPI().get_status(app.owner.pk)
+        status = manager.WorkspaceAPI().get_status(app.workspace_id)
         if not status["is_active"]:
             return False, status["message"]
     return True, None
 
 
-def has_app_permission(user, obj, method):
-    obj = getattr(obj, 'app', obj)
-    has_permission, message = False, f"{obj} object does not exist or does not have permission."
-    if user.is_superuser:
-        has_permission, message = True, None
-    elif getattr(obj, "owner", None) == user:
-        has_permission, message = True, None
-    elif user.is_staff:
-        has_permission, message = True, None
-    else:
-        permission = app.app_permission_registry.get(method)
-        if permission and user.has_perm(permission.codename, obj):
-            has_permission, message = True, None
-    if has_permission and isinstance(obj, app.App):
-        return get_app_status(obj)
-    return has_permission, message
-
-
-class IsAnonymous(permissions.BasePermission):
-    """
-    View permission to allow anonymous users.
-    """
-
-    def has_permission(self, request, view):
-        """
-        Return `True` if permission is granted, `False` otherwise.
-        """
-        return type(request.user) is AnonymousUser
-
-
 class IsOwner(permissions.BasePermission):
     """
     Object-level permission to allow only owners of an object to access it.
@@ -61,54 +30,29 @@ def has_object_permission(self, request, view, obj):
             return False
 
 
-class IsOwnerOrAdmin(permissions.BasePermission):
+class IsAppUser(permissions.BasePermission):
     """
-    Object-level permission to allow only owners of an object or administrators to access it.
-    Assumes the model instance has an `owner` attribute.
+    Object-level permission to allow only users who are owners
+    or collaborators of an app to access it.
     """
-    def has_object_permission(self, request, view, obj):
-        if request.user.is_superuser:
-            return True
-        if hasattr(obj, 'owner'):
-            return obj.owner == request.user
-        else:
-            return False
-
 
-class IsObjectUser(permissions.BasePermission):
-    """
-    Object-level permission to allow owners or collaborators to access
-    an app-related model.
-    """
     def has_object_permission(self, request, view, obj):
-        return has_app_permission(request.user, obj, request.method)[0]
-
-
-class IsAdmin(permissions.BasePermission):
-    """
-    View permission to allow only admins.
-    """
-
-    def has_permission(self, request, view):
-        """
-        Return `True` if permission is granted, `False` otherwise.
-        """
-        return request.user.is_superuser
-
-
-class IsAdminOrSafeMethod(permissions.BasePermission):
-    """
-    View permission to allow only admins to use unsafe methods
-    including POST, PUT, DELETE.
-
-    This allows
-    """
-
-    def has_permission(self, request, view):
-        """
-        Return `True` if permission is granted, `False` otherwise.
-        """
-        return request.method in permissions.SAFE_METHODS or request.user.is_superuser
+        if request.user.is_superuser or request.user.is_staff:
+            return True
+        elif getattr(obj, "user", None) == request.user:
+            return True
+        elif isinstance(obj, Workspace) or hasattr(obj, 'workspace'):
+            workspace = obj if isinstance(obj, Workspace) else obj.workspace
+            if request.method in ["GET", "HEAD", "OPTIONS"]:
+                allowed_roles = ["viewer", "member", "admin"]
+            elif request.method in ["POST", "PUT", "PATCH"]:
+                allowed_roles = ["member", "admin"]
+            else:
+                allowed_roles = ["admin"]
+            return WorkspaceMember.objects.filter(
+                workspace=workspace, user=request.user, role__in=allowed_roles,
+            ).exists()
+        return False
 
 
 class IsServiceToken(permissions.BasePermission):
diff --git a/rootfs/api/serializers/__init__.py b/rootfs/api/serializers/__init__.py
index 6bd1e624f..146736d93 100644
--- a/rootfs/api/serializers/__init__.py
+++ b/rootfs/api/serializers/__init__.py
@@ -201,27 +201,29 @@ class Meta:
 class AppSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.app.App` model."""
 
-    owner = serializers.ReadOnlyField(source='owner.username')
+    workspace = serializers.SlugRelatedField(
+        slug_field='name',
+        queryset=models.workspace.Workspace.objects.all(),
+    )
     structure = serializers.JSONField(required=False)
 
     class Meta:
         """Metadata options for a :class:`AppSerializer`."""
         model = models.app.App
-        fields = ['uuid', 'id', 'owner', 'structure', 'created', 'updated']
+        fields = ['uuid', 'id', 'workspace', 'structure', 'created', 'updated']
 
 
 class BuildSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.build.Build` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     procfile = serializers.JSONField(required=False)
     dryccfile = serializers.JSONField(required=False)
 
     class Meta:
         """Metadata options for a :class:`BuildSerializer`."""
         model = models.build.Build
-        fields = ['owner', 'app', 'image', 'stack', 'sha', 'procfile', 'dryccfile',
+        fields = ['app', 'image', 'stack', 'sha', 'procfile', 'dryccfile',
                   'dockerfile', 'created', 'updated', 'uuid']
 
     @staticmethod
@@ -277,7 +279,6 @@ class ConfigSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.config.Config` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     values = JSONFieldSerializer(required=False, binary=True)
     limits = JSONFieldSerializer(required=False, binary=True)
     lifecycle = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
@@ -419,7 +420,6 @@ class ReleaseSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.release.Release` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
 
     class Meta:
         """Metadata options for a :class:`ReleaseSerializer`."""
@@ -442,12 +442,11 @@ class DomainSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.domain.Domain` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
 
     class Meta:
         """Metadata options for a :class:`DomainSerializer`."""
         model = models.domain.Domain
-        fields = ['owner', 'created', 'updated', 'app', 'domain', 'ptype']
+        fields = ['created', 'updated', 'app', 'domain', 'ptype']
         read_only_fields = ['uuid']
 
     @staticmethod
@@ -504,7 +503,6 @@ class ServiceSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.service.Service` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     port = serializers.IntegerField(required=True)
     protocol = serializers.CharField(required=True)
     target_port = serializers.IntegerField(default=DEFAULT_CONTAINER_PORT)
@@ -513,7 +511,7 @@ class ServiceSerializer(serializers.ModelSerializer):
     class Meta:
         """Metadata options for a :class:`ServiceSerializer`."""
         model = models.service.Service
-        fields = ['owner', 'created', 'updated', 'app', 'ptype']
+        fields = ['created', 'updated', 'app', 'ptype']
         read_only_fields = ['uuid']
 
     @staticmethod
@@ -536,7 +534,6 @@ class CertificateSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.certificate.Certificate` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     domains = serializers.ReadOnlyField()
     san = serializers.ListField(
         child=serializers.CharField(allow_blank=True, allow_null=True, required=False),
@@ -592,7 +589,6 @@ class AppSettingsSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.appsettings.AppSettings` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     autoscale = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
     label = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
 
@@ -615,7 +611,6 @@ class TLSSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.tls.TLS` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     events = serializers.ReadOnlyField()
 
     class Meta:
@@ -628,7 +623,6 @@ class VolumeSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.volume.Volume` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     path = JSONFieldSerializer(required=False, binary=True)
     parameters = serializers.JSONField(required=False)
 
@@ -687,7 +681,6 @@ def validate_parameters(data):
 class ResourceSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.resource.Resource` model."""
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     name = serializers.CharField(max_length=63, required=True)
     plan = serializers.CharField(max_length=128, required=True)
     data = JSONFieldSerializer(required=False, binary=True)
@@ -743,7 +736,6 @@ def validate(self, attrs):
 
 class GatewaySerializer(serializers.Serializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     name = serializers.CharField(max_length=63, required=True)
     listeners = serializers.JSONField(required=False)
     addresses = serializers.JSONField(read_only=True)
@@ -767,7 +759,6 @@ def validate_protocol(value):
 
 class RouteSerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     kind = serializers.CharField(max_length=15, required=True)
     name = serializers.CharField(max_length=63, required=True)
     rules = serializers.JSONField(required=False)
@@ -793,3 +784,43 @@ def validate_rules(self, value):
         else:
             schema = rules.SCHEMA
         return validate_json(value, schema, serializers.ValidationError)
+
+
+class WorkspaceSerializer(serializers.ModelSerializer):
+    """Serialize Workspace model."""
+
+    class Meta:
+        model = models.workspace.Workspace
+        fields = '__all__'
+        read_only_fields = ['id', 'created', 'updated']
+        extra_kwargs = {
+            'name': {'validators': [models.workspace.validate_workspace_name]},
+        }
+
+    def update(self, instance, validated_data):
+        # Workspace name cannot be modified; the name field is read-only after creation.
+        validated_data.pop('name', None)
+        return super().update(instance, validated_data)
+
+
+class WorkspaceMemberSerializer(serializers.ModelSerializer):
+    """Serialize WorkspaceMember model."""
+    user = serializers.ReadOnlyField(source='user.username')
+    email = serializers.ReadOnlyField(source='user.email')
+    workspace = serializers.ReadOnlyField(source='workspace.name')
+
+    class Meta:
+        model = models.workspace.WorkspaceMember
+        fields = '__all__'
+        read_only_fields = ['id', 'created', 'updated']
+
+
+class WorkspaceInvitationSerializer(serializers.ModelSerializer):
+    """Serialize WorkspaceInvitation model."""
+    inviter = serializers.ReadOnlyField(source='inviter.username')
+    workspace = serializers.ReadOnlyField(source='workspace.name')
+
+    class Meta:
+        model = models.workspace.WorkspaceInvitation
+        fields = '__all__'
+        read_only_fields = ['id', 'token', 'created']
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
index e323267c9..4a9183d94 100644
--- a/rootfs/api/settings/celery.py
+++ b/rootfs/api/settings/celery.py
@@ -54,10 +54,6 @@ class Config(object):
             'queue': 'controller.high',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
-        'api.tasks.downstream_model_owner': {
-            'queue': 'controller.high',
-            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
-        },
         'api.tasks.send_app_log': {
             'queue': 'controller.middle',
             'exchange': 'controller.priority', 'routing_key': 'controller.priority.middle',
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 304bc416d..7f2d43b88 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -72,6 +72,7 @@ def randstr(k):
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [
             # insert your TEMPLATE_DIRS here
+            os.path.join(BASE_DIR, "templates"),
         ],
         'APP_DIRS': True,
         'OPTIONS': {
@@ -113,7 +114,6 @@ def randstr(k):
     'django.contrib.sessions',
     # Third-party apps
     'corsheaders',
-    'guardian',
     'gunicorn',
     'rest_framework',
     'social_django',
@@ -126,11 +126,8 @@ def randstr(k):
 
 AUTHENTICATION_BACKENDS = (
     "django.contrib.auth.backends.ModelBackend",
-    "guardian.backends.ObjectPermissionBackend",
     "api.apps_extra.social_core.backends.DryccOIDC",
 )
-GUARDIAN_GET_INIT_ANONYMOUS_USER = 'api.models.base.get_anonymous_user_instance'
-ANONYMOUS_USER_NAME = os.environ.get('ANONYMOUS_USER_NAME', 'AnonymousUser')
 LOGIN_URL = '/v2/auth/login/'
 
 # Security settings
@@ -426,6 +423,10 @@ def randstr(k):
 # Oauth settings
 
 DRYCC_PASSPORT_URL = os.environ.get('DRYCC_PASSPORT_URL', 'https://127.0.0.1:8000')
+DRYCC_REGISTER_URL = os.environ.get(
+    'DRYCC_REGISTER_URL',
+    f'{DRYCC_PASSPORT_URL}/user/registration'
+)
 
 LOGIN_REDIRECT_URL = os.environ.get(
     'LOGIN_REDIRECT_URL',
@@ -459,6 +460,10 @@ def randstr(k):
 )
 DRYCC_CACHE_USER_TIME = int(os.environ.get('DRYCC_CACHE_USER_TIME', 30 * 60))
 
+# Rate limit for invitation emails: max LIMIT emails per WINDOW seconds per recipient address
+DRYCC_INVITATION_EMAIL_LIMIT = int(os.environ.get('DRYCC_INVITATION_EMAIL_LIMIT', 5))
+DRYCC_INVITATION_EMAIL_TIMEOUT = int(os.environ.get('DRYCC_INVITATION_EMAIL_TIMEOUT', 3600))
+
 # Cache Valkey Configuration
 CACHES = {
     "default": {
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index 9cd966f09..3b7e3831b 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -5,6 +5,7 @@
 """
 import time
 import hashlib
+
 import hmac
 import logging
 import urllib.parse
@@ -27,6 +28,7 @@
 from api.models.tls import TLS
 from api.models.volume import Volume
 from api.models.resource import Resource
+from api.models.workspace import WorkspaceInvitation
 
 
 User = get_user_model()
@@ -93,7 +95,7 @@ def _hook_release_created(**kwargs):
                 'release': release.version_name,
                 'release_summary': release.summary,
                 'sha': '',
-                'user': release.owner,
+                'user': release.app.workspace.name,
             }
             if release.build is not None:
                 params['sha'] = release.build.sha
@@ -169,6 +171,13 @@ def tls_changed_handle(sender, instance: TLS, created=False, update_fields=None,
                 route.refresh_to_k8s()
 
 
+@receiver(post_save, sender=User)
+def user_changed_handle(sender, instance: User, created=False, update_fields=None, **kwargs):
+    if created or (update_fields is not None and "email" in update_fields):
+        for invitation in WorkspaceInvitation.objects.filter(email=instance.email, accepted=True):
+            invitation.accept()
+
+
 @receiver(post_save, sender=Gateway)
 def gateway_changed_handle(
         sender, instance: Gateway, created=False, update_fields=None, **kwargs):
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index bd068c6f2..75242e79a 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -5,7 +5,7 @@
 from django.core import signals
 from celery import shared_task
 
-from api import utils, manager, models
+from api import utils, manager
 from api.exceptions import ServiceUnavailable
 logger = logging.getLogger(__name__)
 
@@ -138,28 +138,6 @@ def mount_app(app, user, volume, path):
         signals.request_finished.send(sender=task_id)
 
 
-@shared_task(
-    retry_kwargs={'max_retries': 3}
-)
-def downstream_model_owner(app, old_owner, new_owner):
-    task_id = uuid.uuid4().hex
-    signals.request_started.send(sender=task_id)
-    try:
-        for downstream_model in [
-                models.appsettings.AppSettings, models.build.Build, models.config.Config,
-                models.domain.Domain, models.release.Release, models.resource.Resource,
-                models.tls.TLS, models.service.Service, models.volume.Volume,
-                models.gateway.Gateway, models.gateway.Route]:
-            downstream_model.objects.filter(owner=old_owner, app=app).update(owner=new_owner)
-        app.owner = new_owner
-        app.save()
-    except Exception as e:
-        signals.got_request_exception.send(sender=task_id)
-        raise e
-    else:
-        signals.request_finished.send(sender=task_id)
-
-
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_kwargs={'max_retries': 3}
diff --git a/rootfs/api/templates/workspace/workspace_invitation.html b/rootfs/api/templates/workspace/workspace_invitation.html
new file mode 100644
index 000000000..fd680435b
--- /dev/null
+++ b/rootfs/api/templates/workspace/workspace_invitation.html
@@ -0,0 +1,7 @@
+{% autoescape off %}
+Hi there,
+
+Please click on the link below to confirm joining {{ invitation.workspace.name }} workspace:
+
+{{ domain }}{% url 'workspace_invitation_detail' name=invitation.workspace.name uid=invitation.token %}
+{% endautoescape %}
\ No newline at end of file
diff --git a/rootfs/api/templates/workspace/workspace_invitation_accept.html b/rootfs/api/templates/workspace/workspace_invitation_accept.html
new file mode 100644
index 000000000..94c5c1893
--- /dev/null
+++ b/rootfs/api/templates/workspace/workspace_invitation_accept.html
@@ -0,0 +1,107 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>Workspace Invitation</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style>
+        :root {
+            --vscode-foreground: #CCCCCC;
+            --vscode-editor-background: #1F1F1F;
+            --vscode-textLink-foreground: #4daafc;
+        }
+        @media (prefers-color-scheme: light) {
+            :root {
+                --vscode-foreground: #3B3B3B;
+                --vscode-editor-background: #FFFFFF;
+                --vscode-textLink-foreground: #005FB8;
+            }
+        }
+        html { height: 100%; }
+        body {
+            box-sizing: border-box;
+            min-height: 100%;
+            margin: 0;
+            padding: 15px 30px;
+            display: flex;
+            flex-direction: column;
+            color: var(--vscode-foreground);
+            font-family: -apple-system, BlinkMacSystemFont, "Segoe WPC", "Segoe UI", system-ui, "Ubuntu", "Droid Sans", sans-serif;
+            background-color: var(--vscode-editor-background);
+        }
+        a { color: var(--vscode-textLink-foreground); text-decoration: none; }
+        a:hover, a:focus { text-decoration: underline; }
+        .container {
+            height: 100vh;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            text-align: center;
+        }
+        .title {
+            font-size: 28px;
+            font-weight: 300;
+            margin: 0 0 12px 0;
+            color: var(--vscode-foreground);
+        }
+        .subtitle {
+            font-size: 18px;
+            font-weight: 300;
+            color: var(--vscode-foreground);
+            opacity: 0.7;
+            margin: 0 0 36px 0;
+        }
+        .detail {
+            font-size: 14px;
+            color: var(--vscode-foreground);
+            opacity: 0.7;
+            margin: 0 0 8px 0;
+        }
+        .icon-container {
+            margin-bottom: 24px;
+        }
+        @keyframes rise-and-glow {
+            0% { opacity: 0; transform: translateY(15px) scale(0.95); }
+            50% { opacity: 0.8; transform: translateY(-2px) scale(1.02); }
+            70% { opacity: 1; transform: translateY(1px) scale(0.99); }
+            100% { opacity: 1; transform: translateY(0) scale(1); }
+        }
+        .drycc-icon {
+            width: 128px;
+            height: 128px;
+            animation: rise-and-glow 1.8s cubic-bezier(0.25, 0.46, 0.45, 0.94) forwards;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="content">
+            <div class="icon-container">
+                <svg class="drycc-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16">
+                    <g stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="3" cy="3" r="2"/>
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="3" cy="13" r="2"/>
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="13" cy="13" r="2"/>
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="13" cy="3" r="2"/>
+                        <rect stroke="#3FB1E5" fill="#3FB1E5" x="6.5" y="6.5" width="3" height="3"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" points="8 11 10 15 6 15"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" transform="translate(3,8) rotate(90) translate(-3,-8)" points="3 6 5 10 1 10"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" transform="translate(8,3) rotate(180) translate(-8,-3)" points="8 1 10 5 6 5"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" transform="translate(13,8) rotate(270) translate(-13,-8)" points="13 6 15 10 11 10"/>
+                    </g>
+                </svg>
+            </div>
+            {% if user_exists %}
+            <h1 class="title">Invitation Accepted</h1>
+            <p class="subtitle">You have been added to workspace <strong>{{ workspace_name }}</strong>.</p>
+            <p class="detail">Use the Drycc CLI to log in: <code>drycc login</code></p>
+            {% else %}
+            <h1 class="title">You're Invited to <strong>{{ workspace_name }}</strong></h1>
+            <p class="subtitle">You need an account to join this workspace.</p>
+            <p class="detail">First, <a href="{{ register_url }}">sign up on Drycc Passport</a>.</p>
+            <p class="detail">Then use the Drycc CLI to log in: <code>drycc login</code></p>
+            {% endif %}
+        </div>
+    </div>
+</body>
+</html>
diff --git a/rootfs/api/tests/__init__.py b/rootfs/api/tests/__init__.py
index 835bb4be1..052cfe63c 100644
--- a/rootfs/api/tests/__init__.py
+++ b/rootfs/api/tests/__init__.py
@@ -6,9 +6,13 @@
 from os.path import dirname, realpath
 
 from django.test.runner import DiscoverRunner
+from django.contrib.auth import get_user_model
 from rest_framework.test import APITestCase, APITransactionTestCase
 
 from api.models.base import Token
+from api.models.workspace import Workspace, WorkspaceMember
+
+User = get_user_model()
 
 
 # Mock out router requests and add in some jitter
@@ -56,10 +60,49 @@ def run_tests(self, test_labels, **kwargs):
 
 class DryccBaseTestCase(unittest.TestCase):
 
-    def create_app(self, name=None):
-        body = {}
+    def _get_authenticated_user(self):
+        credentials = getattr(getattr(self, 'client', None), '_credentials', {})
+        auth = credentials.get('HTTP_AUTHORIZATION', '')
+        if auth.startswith('Token '):
+            token_key = auth.split(' ', 1)[1]
+            token = Token.objects.filter(key=token_key).select_related('owner').first()
+            if token is not None:
+                return token.owner
+        return getattr(self, 'user', None)
+
+    def _default_workspace_name(self):
+        user = self._get_authenticated_user()
+        if user and user.username:
+            base = ''.join(ch for ch in user.username.lower() if ch.isalnum())
+            if len(base) >= 5:
+                return base
+        return 'autotest'
+
+    def _ensure_workspace_admin(self, workspace_name):
+        user = self._get_authenticated_user()
+        if user is None:
+            user = User.objects.filter(username='autotest').first()
+        if user is None:
+            raise AssertionError('No test user available for workspace membership')
+
+        workspace, _ = Workspace.objects.get_or_create(
+            name=workspace_name,
+            defaults={'email': user.email or f'{workspace_name}@example.com'},
+        )
+        WorkspaceMember.objects.update_or_create(
+            workspace=workspace,
+            user=user,
+            defaults={'role': 'admin'},
+        )
+        return workspace.name
+
+    def create_app(self, name=None, workspace=None):
+        workspace_name = workspace or self._default_workspace_name()
+        workspace_name = self._ensure_workspace_admin(workspace_name)
+
+        body = {'workspace': workspace_name}
         if name:
-            body = {'id': name}
+            body['id'] = name
 
         response = self.client.post('/v2/apps', body)
         self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 651156eed..162c8ae19 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -15,9 +15,9 @@
 from django.core.cache import cache
 from django.test.utils import override_settings
 
-from api.models.app import App, app_permission_registry
+from api.models.app import App
 from api.models.base import PTYPE_WEB
-from api.models.config import Config
+from api.models.workspace import Workspace, WorkspaceMember
 from scheduler import KubeException, KubeHTTPException
 
 from api.exceptions import DryccException
@@ -45,6 +45,18 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.workspace_name = self._ensure_workspace_admin(self._default_workspace_name())
+
+        original_post = self.client.post
+
+        def _post_with_workspace(path, data=None, *args, **kwargs):
+            if path == '/v2/apps':
+                payload = {} if data is None else dict(data)
+                payload.setdefault('workspace', self.workspace_name)
+                data = payload
+            return original_post(path, data, *args, **kwargs)
+
+        self.client.post = _post_with_workspace
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
@@ -66,7 +78,8 @@ def test_app(self, mock_requests):
 
         body = {'id': 'new'}
         response = self.client.patch(url, body)
-        self.assertEqual(response.status_code, 405, response.content)
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(str(response.data['detail']), 'workspace is required')
 
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
@@ -90,10 +103,10 @@ def test_response_data(self, mock_requests):
         body = {'id': 'app-{}'.format(random.randrange(1000, 10000))}
         response = self.client.post('/v2/apps', body)
         for key in response.data:
-            self.assertIn(key, ['uuid', 'created', 'updated', 'id', 'owner', 'structure'])
+            self.assertIn(key, ['uuid', 'created', 'updated', 'id', 'workspace', 'structure'])
         expected = {
             'id': body['id'],
-            'owner': self.user.username,
+            'workspace': self.user.username,
             'structure': {}
         }
         self.assertEqual(response.data, expected | response.data)
@@ -194,6 +207,13 @@ def test_admin_can_manage_other_apps(self, mock_requests, mock_logger):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
         app_id = self.create_app()
 
+        app = App.objects.get(id=app_id)
+        WorkspaceMember.objects.get_or_create(
+            workspace=app.workspace,
+            user=self.user,
+            defaults={'role': 'admin'},
+        )
+
         # log in as admin, check to see if they have access
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         url = '/v2/apps/{}'.format(app_id)
@@ -216,7 +236,14 @@ def test_admin_can_see_other_apps(self, mock_requests):
         user = User.objects.get(username='autotest2')
         token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
-        self.create_app()
+        app_id = self.create_app()
+
+        app = App.objects.get(id=app_id)
+        WorkspaceMember.objects.get_or_create(
+            workspace=app.workspace,
+            user=self.user,
+            defaults={'role': 'viewer'},
+        )
 
         # log in as admin
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
@@ -288,7 +315,7 @@ def test_unauthorized_user_cannot_see_app(self, mock_requests):
         An unauthorized user should not be able to access an app's resources.
 
         Since an unauthorized user can't access the application, these
-        tests should return a 403, but currently return a 404. FIXME!
+        tests return 404 when app is filtered out from queryset.
         """
         app_id = self.create_app()
         unauthorized_user = User.objects.get(username='autotest2')
@@ -298,14 +325,14 @@ def test_unauthorized_user_cannot_see_app(self, mock_requests):
         url = '/v2/apps/{}/run'.format(app_id)
         body = {'command': 'foo'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
         url = '/v2/apps/{}'.format(app_id)
         response = self.client.get(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
     def test_app_info_not_showing_wrong_app(self, mock_requests):
         self.create_app()
@@ -318,70 +345,78 @@ def test_app_transfer(self, mock_requests):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
 
         collaborator = User.objects.get(username='autotest3')
+        collab_token = self.get_or_create_token(collaborator)
 
-        app = App.objects.create(owner=owner)
-
-        # pretend the owner and a collaborator added some config to the app to ensure
-        # resources owned by the owner are transferred, but not resources owned by the
-        # collaborator.
+        # create a workspace and app under owner
+        response = self.client.post('/v2/workspaces', {
+            'name': 'wstransfer1',
+            'email': 'ws-owner@example.com',
+        })
+        self.assertEqual(response.status_code, 201, response.data)
 
-        config1 = Config.objects.create(
-            owner=owner, app=app, values=[{"name": "FOO", "value": "bar", "group": "global"}])
-        config2 = Config.objects.create(
-            owner=collaborator, app=app,
-            values=[{"name": "CAR", "value": "star", "group": "global"}])
+        response = self.client.post('/v2/apps', {
+            'id': 'app-transfer01',
+            'workspace': 'wstransfer1',
+        })
+        self.assertEqual(response.status_code, 201, response.data)
 
-        # Transfer App
+        app = App.objects.get(id='app-transfer01')
+        workspace = Workspace.objects.get(name='wstransfer1')
         url = '/v2/apps/{}'.format(app.id)
-        new_owner = User.objects.get(username='autotest4')
-        new_owner_token = self.get_or_create_token(new_owner)
-        body = {'owner': new_owner.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
 
-        # Original user can no longer access it
+        # collaborator cannot access app before joining workspace
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
         response = self.client.get(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404, response.data)
 
-        # New owner can access it
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + new_owner_token)
+        # owner adds collaborator to workspace
+        WorkspaceMember.objects.get_or_create(
+            workspace=workspace,
+            user=collaborator,
+            defaults={'role': 'member'},
+        )
+
+        # collaborator can access app after membership
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['owner'], new_owner.username)
 
-        # At this point config1.owner field is still the old owner, but the value in the database
-        # was updated to the new owner when we performed the transfer. The object's updated values
-        # needs to be reloaded from the database to get an accurate idea who owns the object.
-        # https://docs.djangoproject.com/en/dev/ref/models/instances/#django.db.models.Model.refresh_from_db
-        config1.refresh_from_db()
-        config2.refresh_from_db()
+        # collaborator cannot manage workspace members
+        response = self.client.patch(
+            f'/v2/workspaces/{workspace.name}/members/{owner.username}',
+            {'role': 'viewer'},
+        )
+        self.assertEqual(response.status_code, 403, response.data)
+
+    def test_transfer_api_success(self, mock_requests):
+        app_id = self.create_app(name='appxferok')
+        response = self.client.post('/v2/workspaces', {
+            'name': 'wstarget',
+            'email': 'target@example.com',
+        })
+        self.assertEqual(response.status_code, 201, response.data)
 
-        # New owner also is given ownership to all resources owned by the original user, but not
-        # resources created by other users
-        self.assertEqual(config1.owner, new_owner)
-        self.assertEqual(config2.owner, collaborator)
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'wstarget'},
+        )
+        self.assertEqual(response.status_code, 204, response.data)
 
-        # Collaborators can't transfer
-        body = {
-            'username': owner.username,
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
-        response = self.client.post(f'/v2/apps/{app.id}/perms/', body)
-        self.assertEqual(response.status_code, 201, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.workspace.name, 'wstarget')
 
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
-        body = {'owner': self.user.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+    def test_transfer_api_requires_workspace(self, mock_requests):
+        app_id = self.create_app(name='appxferreqws')
+        response = self.client.patch(f'/v2/apps/{app_id}', {})
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(str(response.data['detail']), 'workspace is required')
 
-        # Admins can transfer
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {'owner': self.user.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['owner'], self.user.username)
+    def test_transfer_api_target_workspace_not_found(self, mock_requests):
+        app_id = self.create_app(name='appxfer404')
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'workspace-not-exists'},
+        )
+        self.assertEqual(response.status_code, 404, response.data)
 
     def test_app_exists_in_kubernetes(self, mock_requests):
         """
@@ -585,7 +620,7 @@ def test_get_private_registry_config_bad_registry_location(self, mock_requests):
         self.assertEqual(create, None)
 
     def test_build_env_vars(self, mock_requests):
-        app = App.objects.create(owner=self.user)
+        app = App.objects.create(workspace=Workspace.objects.get(name=self.workspace_name))
         # Make sure an exception is raised when calling without a build available
         with self.assertRaises(DryccException):
             app._build_env_vars(app.release_set.latest(), PTYPE_WEB)
@@ -622,7 +657,7 @@ def test_build_env_vars(self, mock_requests):
             })
 
     def test_gather_app_settings(self, mock_requests):
-        app = App.objects.create(owner=self.user)
+        app = App.objects.create(workspace=Workspace.objects.get(name=self.workspace_name))
         app.save()
         data = {'image': 'autotest/example', 'stack': 'container'}
         url = f"/v2/apps/{app.id}/build"
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 6e30e172a..11820bf19 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -88,10 +88,9 @@ def test_response_data(self, mock_requests):
         response = self.client.post(url, body)
 
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'dockerfile',
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'dockerfile',
                                 'dryccfile', 'image', 'stack', 'procfile', 'sha'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'dockerfile': '',
             'image': 'autotest/example',
diff --git a/rootfs/api/tests/test_certificate.py b/rootfs/api/tests/test_certificate.py
index c7efd8f0f..ccb9d5446 100644
--- a/rootfs/api/tests/test_certificate.py
+++ b/rootfs/api/tests/test_certificate.py
@@ -5,6 +5,12 @@
 from api.models.app import App
 from api.models.certificate import Certificate
 from api.tests import TEST_ROOT, DryccTestCase
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+from datetime import datetime, timedelta
+from pytz import utc
 
 User = get_user_model()
 
@@ -19,7 +25,8 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case')
+        app_id = self.create_app(name='test-app-use-case')
+        self.app = App.objects.get(id=app_id)
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domain = 'autotest.example.com'
 
@@ -33,6 +40,34 @@ def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
+    def _create_certificate_pem(self, common_name, san_names=None):
+        private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        subject = issuer = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'Drycc'),
+            x509.NameAttribute(NameOID.COMMON_NAME, common_name),
+        ])
+        builder = x509.CertificateBuilder().subject_name(subject).issuer_name(issuer)
+        builder = builder.public_key(private_key.public_key())
+        builder = builder.serial_number(x509.random_serial_number())
+        builder = builder.not_valid_before(datetime.now(utc) - timedelta(minutes=1))
+        builder = builder.not_valid_after(datetime.now(utc) + timedelta(days=30))
+
+        if san_names:
+            builder = builder.add_extension(
+                x509.SubjectAlternativeName([x509.DNSName(name) for name in san_names]),
+                critical=False,
+            )
+
+        certificate = builder.sign(private_key=private_key, algorithm=hashes.SHA256())
+        cert_pem = certificate.public_bytes(serialization.Encoding.PEM).decode('utf-8')
+        key_pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode('utf-8')
+        return cert_pem, key_pem, certificate
+
     def test_create_certificate_with_domain(self):
         """Tests creating a certificate."""
         response = self.client.post(
@@ -146,7 +181,6 @@ def test_delete_certificate(self):
         Certificate.objects.create(
             name='random-test-cert',
             app=self.app,
-            owner=self.user,
             common_name='autotest.example.com',
             certificate=self.cert,
             key=self.key
@@ -156,7 +190,7 @@ def test_delete_certificate(self):
         self.assertEqual(response.status_code, 204, response.data)
 
     def test_create_invalid_cert(self):
-        """Upload a cert that can't be loaded by pyopenssl"""
+        """Upload a cert that can't be parsed."""
         response = self.client.post(
             self.url,
             {
@@ -166,15 +200,14 @@ def test_create_invalid_cert(self):
             }
         )
         self.assertEqual(response.status_code, 400, response.data)
-        # match partial since right now the rest is pyopenssl errors
+        # Match partial since parser details may vary.
         self.assertIn('Could not load certificate', response.data['certificate'][0])
 
     def test_load_invalid_cert(self):
-        """Inject a cert that can't be loaded by pyopenssl"""
+        """Inject a cert that can't be parsed."""
 
         with self.assertRaises(SuspiciousOperation):
             Certificate.objects.create(
-                owner=self.user,
                 app=self.app,
                 name='random-test-cert',
                 certificate='i am bad data',
@@ -182,7 +215,7 @@ def test_load_invalid_cert(self):
             )
 
     def test_create_invalid_key(self):
-        """Upload a private key that can't be loaded by pyopenssl"""
+        """Upload a private key that can't be parsed."""
         response = self.client.post(
             self.url,
             {
@@ -192,20 +225,71 @@ def test_create_invalid_key(self):
             }
         )
         self.assertEqual(response.status_code, 400, response.data)
-        # match partial since right now the rest is pyopenssl errors
+        # Match partial since parser details may vary.
         self.assertIn('Could not load private key', response.data['key'][0])
 
     def test_load_invalid_key(self):
-        """Inject a private key that can't be loaded by pyopenssl"""
+        """Inject a private key that can't be parsed."""
 
         with self.assertRaises(SuspiciousOperation):
             Certificate.objects.create(
-                owner=self.user,
+                app=self.app,
                 name='random-test-cert',
                 certificate=self.cert,
                 key='I am Groot.'
             )
 
+    def test_create_certificate_with_san(self):
+        """Certificates with SAN should expose DNS names."""
+        cert, key, _ = self._create_certificate_pem(
+            'autotest.example.com',
+            san_names=['autotest.example.com', 'www.autotest.example.com'],
+        )
+
+        response = self.client.post(
+            self.url,
+            {
+                'name': 'san-test-cert',
+                'certificate': cert,
+                'key': key,
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(
+            response.data['san'],
+            ['autotest.example.com', 'www.autotest.example.com'],
+        )
+
+    def test_create_certificate_with_mismatched_key(self):
+        """Certificate and private key must match."""
+        cert, _, _ = self._create_certificate_pem('autotest.example.com')
+        _, other_key, _ = self._create_certificate_pem('other.example.com')
+
+        response = self.client.post(
+            self.url,
+            {
+                'name': 'mismatch-test-cert',
+                'certificate': cert,
+                'key': other_key,
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertIn('Certificate and private key do not match!', response.data[0])
+
+    def test_certificate_persists_subject_and_issuer(self):
+        """Persisted certificate metadata should reflect x509 subject and issuer."""
+        cert, key, parsed = self._create_certificate_pem('autotest.example.com')
+
+        certificate = Certificate.objects.create(
+            name='metadata-test-cert',
+            app=self.app,
+            certificate=cert,
+            key=key,
+        )
+
+        self.assertEqual(certificate.subject, parsed.subject.rfc4514_string())
+        self.assertEqual(certificate.issuer, parsed.issuer.rfc4514_string())
+
     def test_certs_fetch_limit(self):
         """
         When a user retrieves a certificate, make sure limits work
diff --git a/rootfs/api/tests/test_certificate_use_case_1.py b/rootfs/api/tests/test_certificate_use_case_1.py
index ea6d5a64a..604a9f4b4 100644
--- a/rootfs/api/tests/test_certificate_use_case_1.py
+++ b/rootfs/api/tests/test_certificate_use_case_1.py
@@ -24,10 +24,11 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-1')
+        app_id = self.create_app(name='test-app-use-case-1')
+        self.app = App.objects.get(id=app_id)
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domain = Domain.objects.create(
-            owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB)
+            app=self.app, domain='foo.com', ptype=PTYPE_WEB)
         self.name = 'foo-com'  # certificate name
 
         with open('{}/certs/{}.key'.format(TEST_ROOT, self.domain)) as f:
@@ -147,7 +148,6 @@ def test_certficate_denied_requests(self):
     def test_delete_certificate(self):
         """Destroying a certificate should generate a 204 response"""
         Certificate.objects.create(
-            owner=self.user,
             app=self.app,
             name=self.name,
             certificate=self.cert,
diff --git a/rootfs/api/tests/test_certificate_use_case_2.py b/rootfs/api/tests/test_certificate_use_case_2.py
index 7f808f890..c74845291 100644
--- a/rootfs/api/tests/test_certificate_use_case_2.py
+++ b/rootfs/api/tests/test_certificate_use_case_2.py
@@ -24,13 +24,14 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-2')
+        app_id = self.create_app(name='test-app-use-case-2')
+        self.app = App.objects.get(id=app_id)
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             'foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB),
+                app=self.app, domain='foo.com', ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         # only foo.com has a cert
@@ -133,7 +134,6 @@ def test_delete_certificate(self):
             Certificate.objects.create(
                 name=certificate['name'],
                 app=self.app,
-                owner=self.user,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
diff --git a/rootfs/api/tests/test_certificate_use_case_3.py b/rootfs/api/tests/test_certificate_use_case_3.py
index 9d17a2f97..7eedfc859 100644
--- a/rootfs/api/tests/test_certificate_use_case_3.py
+++ b/rootfs/api/tests/test_certificate_use_case_3.py
@@ -24,13 +24,14 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-3')
+        app_id = self.create_app(name='test-app-use-case-3')
+        self.app = App.objects.get(id=app_id)
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             'foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB),
+                app=self.app, domain='foo.com', ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
@@ -135,7 +136,6 @@ def test_delete_certificate(self):
             Certificate.objects.create(
                 name=certificate['name'],
                 app=self.app,
-                owner=self.user,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
diff --git a/rootfs/api/tests/test_certificate_use_case_4.py b/rootfs/api/tests/test_certificate_use_case_4.py
index f12b7634b..573cfa571 100644
--- a/rootfs/api/tests/test_certificate_use_case_4.py
+++ b/rootfs/api/tests/test_certificate_use_case_4.py
@@ -24,16 +24,17 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-3')
+        app_id = self.create_app(name='test-app-use-case-3')
+        self.app = App.objects.get(id=app_id)
         self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
             '*.foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='*.foo.com',
+                app=self.app, domain='*.foo.com',
                 ptype=PTYPE_WEB),
             'foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB),
+                app=self.app, domain='foo.com', ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
@@ -218,7 +219,6 @@ def test_delete_certificate(self):
         for domain, certificate in self.certificates.items():
             Certificate.objects.create(
                 name=certificate['name'],
-                owner=self.user,
                 app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
diff --git a/rootfs/api/tests/test_certificate_use_case_5.py b/rootfs/api/tests/test_certificate_use_case_5.py
index 34c331e3c..1a060421b 100644
--- a/rootfs/api/tests/test_certificate_use_case_5.py
+++ b/rootfs/api/tests/test_certificate_use_case_5.py
@@ -24,17 +24,18 @@ def setUp(self):
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-5')
+        app_id = self.create_app(name='test-app-use-case-5')
+        self.app = App.objects.get(id=app_id)
         self.url = f'/v2/apps/{self.app.id}/certs'
         # Done out of scope as it gets the same cert as the wildcard
         Domain.objects.create(
-            owner=self.user, app=self.app, domain='foo.com', ptype=PTYPE_WEB)
+            app=self.app, domain='foo.com', ptype=PTYPE_WEB)
         self.domains = {
             '*.foo.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='*.foo.com',
+                app=self.app, domain='*.foo.com',
                 ptype=PTYPE_WEB),
             'bar.com': Domain.objects.create(
-                owner=self.user, app=self.app, domain='bar.com', ptype=PTYPE_WEB),
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
@@ -164,7 +165,6 @@ def test_delete_certificate(self):
             # Create certificate
             Certificate.objects.create(
                 name=certificate['name'],
-                owner=self.user,
                 app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 22a435fea..c9579df42 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -35,11 +35,8 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-        url = '/v2/apps'
-        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
-        self.assertEqual(response.status_code, 201, response.data)
-        self.app = App.objects.all()[0]
+        app_id = self.create_app()
+        self.app = App.objects.get(id=app_id)
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
@@ -366,11 +363,10 @@ def test_response_data(self, mock_requests):
         body = {'values': [value1]}
         response = self.client.post(url, body)
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values',
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'values',
                                 'values_refs', 'limits', 'tags', 'registry', 'healthcheck',
                                 'lifecycle', 'termination_grace_period'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'values': [value1],
             'limits': {
@@ -397,11 +393,10 @@ def test_response_data_types_converted(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'limits',
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'values', 'limits',
                                 'values_refs', 'tags', 'registry', 'healthcheck',
                                 'lifecycle', 'termination_grace_period'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'values': [value1],
             'limits': {
@@ -563,14 +558,6 @@ def test_admin_can_create_config_on_other_apps(self, mock_requests):
         self.assertEqual([value1], response.data['values'])
         return response
 
-    def test_config_owner_is_requesting_user(self, mock_requests):
-        """
-        Ensure that setting the config value is owned by the requesting user
-        See https://github.com/drycc/drycc/issues/2650
-        """
-        response = self.test_admin_can_create_config_on_other_apps()
-        self.assertEqual(response.data['owner'], self.user.username)
-
     def test_unauthorized_user_cannot_modify_config(self, mock_requests):
         """
         An unauthorized user should not be able to modify other config.
@@ -685,9 +672,7 @@ def test_unset_limits_error(self, mock_requests):
         self.assertEqual(response.status_code, 201)
         # dockerfile + procfile worflow
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
         build = Build.objects.create(
-            owner=user,
             app=app,
             image="qwerty",
             procfile={
@@ -700,7 +685,6 @@ def test_unset_limits_error(self, mock_requests):
         # create an initial release
         release = Release.objects.create(
             version=3,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
@@ -764,9 +748,7 @@ def test_set_config_limits_run(self, *args, **kwargs):
         app_id = self.create_app()
         # dockerfile + procfile worflow
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
         build = Build.objects.create(
-            owner=user,
             app=app,
             image="qwerty",
             procfile={
@@ -779,7 +761,6 @@ def test_set_config_limits_run(self, *args, **kwargs):
         # create an initial release
         release = Release.objects.create(
             version=3,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
diff --git a/rootfs/api/tests/test_consumers.py b/rootfs/api/tests/test_consumers.py
index 244edb3e8..15d84fe34 100644
--- a/rootfs/api/tests/test_consumers.py
+++ b/rootfs/api/tests/test_consumers.py
@@ -421,7 +421,6 @@ def setUp(self):
         # We'll patch the save method to prevent k8s interactions
         with patch.object(Volume, 'save_to_k8s'), patch.object(Volume, 'delete_from_k8s'):
             self.test_volume = Volume.objects.create(
-                owner=self.user,
                 app=self.app,
                 name='test-volume',
                 size='1G',
diff --git a/rootfs/api/tests/test_domain.py b/rootfs/api/tests/test_domain.py
index 7b310ca8d..918eea36d 100644
--- a/rootfs/api/tests/test_domain.py
+++ b/rootfs/api/tests/test_domain.py
@@ -45,10 +45,9 @@ def test_response_data(self):
 
         for key in response.data:
             self.assertIn(
-                key, ['uuid', 'owner', 'created', 'updated', 'app', 'domain', 'ptype'])
+                key, ['uuid', 'created', 'updated', 'app', 'domain', 'ptype'])
 
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'domain': 'test-domain.example.com'
         }
diff --git a/rootfs/api/tests/test_event.py b/rootfs/api/tests/test_event.py
index cf44d61b2..2b3b0d717 100644
--- a/rootfs/api/tests/test_event.py
+++ b/rootfs/api/tests/test_event.py
@@ -25,11 +25,7 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-        url = '/v2/apps'
-        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
-        self.assertEqual(response.status_code, 201, response.data)
-        self.app = App.objects.all()[0]
+        self.app = App.objects.get(id=self.create_app())
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 39f184997..dda6e3119 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -43,13 +43,11 @@ def create_app_with_domain_and_deploy(self):
         self.assertEqual(len(response.data["results"]), 0, response.data)
         # create a release so we can scale
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
-        build = Build.objects.create(owner=user, app=app, image="qwerty")
+        build = Build.objects.create(app=app, image="qwerty")
 
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
@@ -152,7 +150,6 @@ def test_add_listener(self):
                 break
         expect = {
             "app": app_id,
-            "owner": "autotest",
             "name": gateway_name,
             "listeners": [
                 {
@@ -195,7 +192,7 @@ def test_add_tls_listener(self):
         name = "tls-gateway"
         _, domain, secret_name, results = self.add_tls_listener(name, "TLS", port)
         expect = [{
-            'app': name, 'owner': 'autotest', 'name': name,
+            'app': name, 'name': name,
             'listeners': [{
                 'allowedRoutes': {'namespaces': {'from': 'All'}},
                 'name': 'tcp-443-1', 'port': 443, 'hostname': domain,
@@ -213,7 +210,6 @@ def test_add_https_listener(self):
         _, domain, secret_name, results = self.add_tls_listener(name, "HTTPS", port)
         expect = [{
             'app': name,
-            'owner': 'autotest',
             'name': name,
             'listeners': [{
                 'allowedRoutes': {
@@ -244,7 +240,7 @@ def test_add_http_listener(self):
         name = "bingo-gateway"
         _, domain, secret_name, results = self.add_tls_listener(name, "HTTP", port)
         expect = [{
-            'app': 'bingo-gateway', 'owner': 'autotest', 'name': 'bingo-gateway',
+            'app': 'bingo-gateway', 'name': 'bingo-gateway',
             'listeners': [
                 {
                     'allowedRoutes': {'namespaces': {'from': 'All'}},
@@ -280,7 +276,6 @@ def test_add_udp_listener(self):
         _, results = self.add_listener(app_id, name, "UDP", port)
         expect = [{
             'app': 'bingo-gateway',
-            'owner': 'autotest',
             'name': 'bingo-gateway',
             'listeners': [{
                 'allowedRoutes': {
@@ -376,7 +371,6 @@ def test_change_tls(self):
         self.assertEqual(response.data["count"], 1, response.data)
         expect = [{
             'app': app_id,
-            'owner': 'autotest',
             'name': app_id,
             'listeners': [
                 {
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index d528d16b2..970ec5b5d 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -8,7 +8,6 @@
 from django.core.cache import cache
 
 from api.tests import adapter, DryccTransactionTestCase
-from api.models.app import app_permission_registry
 import requests_mock
 
 User = get_user_model()
@@ -70,15 +69,6 @@ def test_key_hook(self, mock_requests):
         # Create app to use
         app_id = self.create_app()
 
-        # give user permission to app
-        body = {
-            'username': str(self.user),
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
-        url = f'/v2/apps/{app_id}/perms/'
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-
         # Create rsa key
         body = {'id': str(self.user), 'public': RSA_PUBKEY}
         response = self.client.post('/v2/keys', body)
diff --git a/rootfs/api/tests/test_perm.py b/rootfs/api/tests/test_perm.py
index 21055e2cf..96ec20e7a 100644
--- a/rootfs/api/tests/test_perm.py
+++ b/rootfs/api/tests/test_perm.py
@@ -1,319 +1,191 @@
 from django.test import tag
 from django.contrib.auth import get_user_model
+
 from api.tests import DryccTestCase
-from api.models.app import App, app_permission_registry, \
-    VIEW_APP_PERMISSION, CHANGE_APP_PERMISSION
+from api.models.workspace import WorkspaceMember
+from api.models.app import App
+
 
 User = get_user_model()
 
 
-class TestUserPerm(DryccTestCase):
+class TestWorkspacePerm(DryccTestCase):
 
-    fixtures = ['test_sharing.json']
+    fixtures = ['tests.json']
 
     @tag('auth')
     def setUp(self):
-        self.user = User.objects.get(username='autotest-1')
+        self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
-        # Always have first user authed coming into tests
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.user2 = User.objects.get(username='autotest-2')
+        self.user2 = User.objects.get(username='autotest2')
         self.token2 = self.get_or_create_token(self.user2)
-        self.user3 = User.objects.get(username='autotest-3')
-        self.token3 = self.get_or_create_token(self.user3)
 
     @tag('auth')
-    def test_create(self):
-        # check that user 1 sees her lone app and user 2's app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-        app_id = response.data['results'][0]['id']
-
-        # check that user 2 can only see his app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(len(response.data['results']), 1)
-        # check that user 2 can't see any of the app's builds, configs,
-        # containers, limits, or releases
-        for model in ['builds', 'config', 'pods', 'releases']:
-            response = self.client.get("/v2/apps/{}/{}/".format(app_id, model))
-            msg = "Failed: status '%s', and data '%s'" % (response.status_code, response.data)
-            self.assertEqual(response.status_code, 403, msg=msg)
-            self.assertEqual(response.data['detail'],
-                             'You do not have permission to perform this action.', msg=msg)
-
-        for app in App.objects.filter(owner=self.user):
-            body = {
-                'username': self.user2.username,
-                'permissions': ','.join(app_permission_registry.shortnames),
-            }
-            self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-            response = self.client.post(f'/v2/apps/{app.id}/perms/', body)
-            self.assertEqual(response.status_code, 201, response.data)
-
-        # check that user 2 can see the app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
+    def _create_workspace_and_app(self, ws_name='testws01', app_id='testapp01'):
+        response = self.client.post('/v2/workspaces', {
+            'name': ws_name,
+            'email': 'ws@example.com',
+        })
+        self.assertEqual(response.status_code, 201, response.data)
 
-        # check that user 2 sees (empty) results now for builds, containers,
-        # and releases. (config and limit will still give 404s since we didn't
-        # push a build here.)
-        for model in ['builds', 'releases']:
-            response = self.client.get("/v2/apps/{}/{}/".format(app_id, model))
-            self.assertEqual(len(response.data['results']), 0)
-        # TODO:  check that user 2 can git push the app
+        response = self.client.post('/v2/apps', {
+            'id': app_id,
+            'workspace': ws_name,
+        })
+        self.assertEqual(response.status_code, 201, response.data)
+        return ws_name, app_id
 
     @tag('auth')
-    def test_create_errors(self):
-        # check that user 1 sees her lone app
-        response = self.client.get('/v2/apps')
-        app_id = response.data['results'][0]['id']
-
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
+    def test_workspace_member_can_access_app(self):
+        ws_name, app_id = self._create_workspace_and_app()
 
-        # check that user 2 can't create a permission
+        # non-member cannot access app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 403)
+        response = self.client.get(f'/v2/apps/{app_id}')
+        self.assertEqual(response.status_code, 404, response.data)
 
-    @tag('auth')
-    def test_delete(self):
-        # give user 2 permission to user 1's app
-        response = self.client.get('/v2/apps')
-        app_id = response.data['results'][0]['id']
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join(
-                [VIEW_APP_PERMISSION.shortname, CHANGE_APP_PERMISSION.shortname]),
-        }
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 201, response.data)
+        # add user2 into workspace, then user2 can access app
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # check that user 2 can see the app as well as his own
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
+        response = self.client.get(f'/v2/apps/{app_id}')
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
 
-        # delete permission to user 1's app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        url = f"/v2/apps/{app_id}/perms/{self.user2.username}/"
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204, response)
-        self.assertIsNone(response.data)
+    @tag('auth')
+    def test_non_admin_cannot_manage_workspace_members(self):
+        ws_name, _ = self._create_workspace_and_app(ws_name='testws02', app_id='testapp02')
 
-        # check that user 2 can only see his app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(len(response.data['results']), 1)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # delete permission to user 1's app again, expecting an error
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.patch(
+            f'/v2/workspaces/{ws_name}/members/{self.user.username}',
+            {'role': 'viewer'},
+        )
+        self.assertEqual(response.status_code, 403, response.data)
 
     @tag('auth')
-    def test_list(self):
-        # check that user 1 sees her lone app and user 2's app
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-        app_id = response.data['results'][0]['id']
-
-        # create a new object permission
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 201, response.data)
+    def test_admin_can_manage_workspace_members(self):
+        ws_name, _ = self._create_workspace_and_app(ws_name='testws03', app_id='testapp03')
 
-        # list perms on the app
-        response = self.client.get(f"/v2/apps/{app_id}/perms/")
-        self.assertEqual(response.data["count"], 1)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-    @tag('auth')
-    def test_admin_can_list(self):
-        """Check that an administrator can list an app's perms"""
-        response = self.client.get('/v2/apps')
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        response = self.client.patch(
+            f'/v2/workspaces/{ws_name}/members/{self.user2.username}',
+            {'role': 'viewer'},
+        )
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
+        self.assertEqual(response.data['role'], 'viewer')
 
     @tag('auth')
-    def test_list_errors(self):
-        response = self.client.get('/v2/apps')
-        # login as user 2, list perms on the app
+    def test_non_member_cannot_run_app(self):
+        _, app_id = self._create_workspace_and_app(ws_name='testws04', app_id='testapp04')
+
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get(f"/v2/apps/{self.user2.id}/perms/")
-        self.assertEqual(response.status_code, 404)
+        response = self.client.post(
+            f'/v2/apps/{app_id}/run',
+            {'command': 'echo hello'},
+        )
+        self.assertEqual(response.status_code, 404, response.data)
 
     @tag('auth')
-    def test_unauthorized_user_cannot_modify_perms(self):
-        """
-        An unauthorized user should not be able to modify other apps' permissions.
-
-        Since an unauthorized user should not know about the application at all, these
-        requests should return a 404.
-        """
-        app_id = 'autotest'
-        url = '/v2/apps'
-        body = {'id': app_id}
-        response = self.client.post(url, body)
-
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
+    def test_workspace_member_can_run_but_without_build_gets_business_error(self):
+        ws_name, app_id = self._create_workspace_and_app(ws_name='testws05', app_id='testapp05')
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
+
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 403)
+        response = self.client.post(
+            f'/v2/apps/{app_id}/run',
+            {'command': 'echo hello'},
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(
+            str(response.data['detail']),
+            'no build available, please deploy a release',
+        )
 
     @tag('auth')
-    def test_collaborator_share(self):
-        """
-        An collaborator should not be able to modify the app's permissions.
-        """
-        app_id = "autotest-1-app"
-        owner_token = self.token
-        collab = self.user2
-        collab_token = self.token2
-
-        # Share app with collaborator
-        body = {
-            'username': collab.username,
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
-        url = f'/v2/apps/{app_id}/perms/'
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
+    def test_non_admin_cannot_update_workspace(self):
+        ws_name, _ = self._create_workspace_and_app(ws_name='testws06', app_id='testapp06')
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # Collaborator can share app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
-        body = {
-            'username': self.user3.username,
-            'permissions': ','.join(app_permission_registry.shortnames),
-        }
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201)
-
-        # Collaborator can list
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.patch(
+            f'/v2/workspaces/{ws_name}',
+            {'email': 'new@example.com'},
+        )
+        self.assertEqual(response.status_code, 403, response.data)
 
-        # Share app with user 3 for rest of tests
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
-        response = self.client.post(url, body)
+    @tag('auth')
+    def test_non_admin_cannot_transfer_app(self):
+        ws_name, app_id = self._create_workspace_and_app(ws_name='testws07', app_id='testapp07')
+        response = self.client.post('/v2/workspaces', {
+            'name': 'testws08',
+            'email': 'ws2@example.com',
+        })
         self.assertEqual(response.status_code, 201, response.data)
 
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token3)
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 2, response.data)
-
-        # Collaborator cannot delete other collaborator
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
-        url += "{}/".format(self.user3.username)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204)
-        url = f'/v2/apps/{app_id}/perms/'
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 1, response.data)
-
-        # Collaborator can delete themselves
-        url += "{}/".format(self.user2.username)
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204)
-        url = f'/v2/apps/{app_id}/perms/'
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 0, response.data)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-    @tag('auth')
-    def test_each_permission(self):
-        # check that user 1 sees her lone app and user 2's app
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-        app_id = response.data['results'][0]['id']
-
-        # list no perms on the app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get(f"/v2/apps/{app_id}/perms/")
-        self.assertEqual(response.status_code, 403)
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'testws08'},
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(
+            str(response.data['detail']),
+            'you must be an admin of the current workspace',
+        )
 
-        # create a new object permission
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
-        }
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
+    @tag('auth')
+    def test_admin_can_transfer_app(self):
+        _, app_id = self._create_workspace_and_app(ws_name='testws09', app_id='testapp09')
+        response = self.client.post('/v2/workspaces', {
+            'name': 'testws10',
+            'email': 'ws10@example.com',
+        })
         self.assertEqual(response.status_code, 201, response.data)
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get(f"/v2/apps/{app_id}/perms/")
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data['count'], 1, response.data)
-
-        # test no change permission
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
-        }
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 403, response.data)
 
-        # test add change permission
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join(
-                [VIEW_APP_PERMISSION.shortname, CHANGE_APP_PERMISSION.shortname]),
-        }
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 201)
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join(
-                [VIEW_APP_PERMISSION.shortname, CHANGE_APP_PERMISSION.shortname]),
-        }
-        response = self.client.post(f'/v2/apps/{app_id}/perms/', body)
-        self.assertEqual(response.status_code, 201)
-
-        # test no delete permission
-        response = self.client.delete(f'/v2/apps/{app_id}/perms/{self.user2.username}/')
-        self.assertEqual(response.status_code, 403)
-
-        # test update permisssion
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
-        }
-        response = self.client.put(f'/v2/apps/{app_id}/perms/{self.user2.username}/', body)
-        self.assertEqual(response.status_code, 204)
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'testws10'},
+        )
+        self.assertEqual(response.status_code, 204, response.data)
 
-        # test not update permission
-        body = {
-            'username': self.user2.username,
-            'permissions': ','.join([VIEW_APP_PERMISSION.shortname]),
-        }
-        response = self.client.put(f'/v2/apps/{app_id}/perms/{self.user2.username}/', body)
-        self.assertEqual(response.status_code, 403)
-
-        # has view permission
-        response = self.client.get(f"/v2/apps/{app_id}/perms/")
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(response.data['count'], 1, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.workspace.name, 'testws10')
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index 5b8a1266a..371b8208c 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -265,13 +265,11 @@ def test_container_errors(self, mock_requests):
 
         # create a release so we can scale
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
-        build = Build.objects.create(owner=user, app=app, image="qwerty")
+        build = Build.objects.create(app=app, image="qwerty")
 
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
@@ -474,9 +472,7 @@ def test_admin_can_manage_other_pods(self, mock_requests):
     def test_scale_without_build_should_error(self, mock_requests):
         """A user should not be able to scale processes unless a build is present."""
         app_id = 'autotest'
-        url = '/v2/apps'
-        body = {'cluster': 'autotest', 'id': app_id}
-        response = self.client.post(url, body)
+        self.create_app(name=app_id)
 
         url = f'/v2/apps/{app_id}/ptypes/scale'
         body = {'web': '1'}
@@ -489,11 +485,9 @@ def test_command_good(self, mock_requests):
         """Test the default command for each container workflow"""
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
 
         # CNCF Buildpack app
         build = Build.objects.create(
-            owner=user,
             app=app,
             image="qwerty",
             procfile={
@@ -507,7 +501,6 @@ def test_command_good(self, mock_requests):
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
@@ -551,7 +544,6 @@ def test_run_command_good(self, mock_requests):
 
         # dockerfile + procfile worflow
         build = Build.objects.create(
-            owner=self.user,
             app=app,
             image="qwerty",
             stack="heroku-18",
@@ -566,7 +558,6 @@ def test_run_command_good(self, mock_requests):
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=self.user,
             app=app,
             config=app.config_set.latest(),
             build=build
@@ -613,7 +604,6 @@ def test_run_not_fail_on_debug(self, mock_requests):
 
         # dockerfile + procfile worflow
         build = Build.objects.create(
-            owner=self.user,
             app=app,
             image="qwerty",
             procfile={
@@ -627,7 +617,6 @@ def test_run_not_fail_on_debug(self, mock_requests):
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=self.user,
             app=app,
             config=app.config_set.latest(),
             build=build
@@ -646,11 +635,9 @@ def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
         """Test that app info doesn't show transient "run" proctypes."""
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
 
         # dockerfile + procfile worflow
         build = Build.objects.create(
-            owner=user,
             app=app,
             image="qwerty",
             procfile={
@@ -664,7 +651,6 @@ def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
index 64749c3fe..15a509bee 100644
--- a/rootfs/api/tests/test_ptypes.py
+++ b/rootfs/api/tests/test_ptypes.py
@@ -25,11 +25,7 @@ def setUp(self):
         self.user = User.objects.get(username='autotest')
         self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-        url = '/v2/apps'
-        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
-        self.assertEqual(response.status_code, 201, response.data)
-        self.app = App.objects.all()[0]
+        self.app = App.objects.get(id=self.create_app())
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 6330ccdb7..88ef6d6e1 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -257,11 +257,10 @@ def test_response_data(self, mock_requests):
         url = '/v2/apps/{}/releases/v2'.format(app_id)
         response = self.client.get(url)
         for key in response.data.keys():
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'build', 'config',
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'build', 'config',
                                 'summary', 'version', 'state', 'failed', 'conditions',
                                 'exception', 'deployed_ptypes'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'build': None,
             'config': uuid.UUID(config_response.data['uuid']),
@@ -735,7 +734,6 @@ def test_release_external_registry(self, mock_requests):
     def test_run(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
         dryccfile = {
             "pipeline": {
                 "web.yaml": {
@@ -767,10 +765,10 @@ def test_run(self, mock_requests):
             },
         }
         build = Build.objects.create(
-            owner=user, app=app, image="qwerty", procfile={},
+            app=app, image="qwerty", procfile={},
             sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
         release = Release.objects.create(
-            version=2, owner=user, app=app, config=app.config_set.latest(),
+            version=2, app=app, config=app.config_set.latest(),
             build=build, state="succeed")
         runners = release.get_runners(["web"])
         self.assertEqual(len(runners), 1)
@@ -786,10 +784,10 @@ def test_run(self, mock_requests):
             'image': 'registry.drycc.cc/drycc/python-dev:latest',
         }
         build = Build.objects.create(
-            owner=user, app=app, image="qwerty", procfile={},
+            app=app, image="qwerty", procfile={},
             sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
         release = Release.objects.create(
-            version=3, owner=user, app=app, config=app.config_set.latest(),
+            version=3, app=app, config=app.config_set.latest(),
             build=build, state="succeed")
         runners = release.get_runners(["web"])
         self.assertEqual(len(runners), 1)
diff --git a/rootfs/api/tests/test_resource.py b/rootfs/api/tests/test_resource.py
index 31a1c4817..37af38ef9 100644
--- a/rootfs/api/tests/test_resource.py
+++ b/rootfs/api/tests/test_resource.py
@@ -61,11 +61,10 @@ def test_resources_create(self, mock_requests):
 
         for key in response.data:
             self.assertIn(key,
-                          ['uuid', 'owner', 'created', 'updated', 'app', 'plan',
+                          ['uuid', 'created', 'updated', 'app', 'plan',
                            'options', 'data', 'status', 'binding', 'name'])
 
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'name': 'mysql',
             'plan': 'mysql:5.6'
diff --git a/rootfs/api/tests/test_users.py b/rootfs/api/tests/test_users.py
deleted file mode 100644
index c8daf13cc..000000000
--- a/rootfs/api/tests/test_users.py
+++ /dev/null
@@ -1,47 +0,0 @@
-from django.contrib.auth import get_user_model
-from api.tests import DryccTestCase
-
-User = get_user_model()
-
-
-class TestUsers(DryccTestCase):
-    """ Tests users endpoint"""
-
-    fixtures = ['tests.json']
-
-    def test_super_user_can_list(self):
-        user = User.objects.get(username='autotest')
-        token = self.get_or_create_token(user)
-
-        for url in ['/v2/users', '/v2/users/']:
-            response = self.client.get(url,
-                                       HTTP_AUTHORIZATION='token {}'.format(token))
-            self.assertEqual(response.status_code, 200, response.data)
-            self.assertEqual(len(response.data['results']), 4)
-
-    def test_enable(self):
-        user = User.objects.get(username='autotest')
-        token = self.get_or_create_token(user)
-        response = self.client.patch("/v2/users/autotest2/enable/",
-                                     HTTP_AUTHORIZATION='token {}'.format(token))
-        self.assertEqual(response.status_code, 204)
-        user = User.objects.get(username='autotest2')
-        self.assertEqual(user.is_active, True)
-
-    def test_disable(self):
-        user = User.objects.get(username='autotest')
-        token = self.get_or_create_token(user)
-        response = self.client.patch("/v2/users/autotest2/disable/",
-                                     HTTP_AUTHORIZATION='token {}'.format(token))
-        self.assertEqual(response.status_code, 204)
-        user = User.objects.get(username='autotest2')
-        self.assertEqual(user.is_active, False)
-
-    def test_non_super_user_cannot_list(self):
-        user = User.objects.get(username='autotest2')
-        token = self.get_or_create_token(user)
-
-        for url in ['/v2/users', '/v2/users/']:
-            response = self.client.get(url,
-                                       HTTP_AUTHORIZATION='token {}'.format(token))
-            self.assertEqual(response.status_code, 403)
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index 0ce9b9f7e..20f908117 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -93,11 +93,10 @@ def test_volumecreate(self, mock_requests):
 
         for key in response.data:
             self.assertIn(key,
-                          ['uuid', 'owner', 'created', 'updated', 'app', 'name',
+                          ['uuid', 'created', 'updated', 'app', 'name',
                            'size', 'path', 'type', 'parameters'])
 
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'name': 'myvolume',
             'size': '500G'
@@ -144,7 +143,6 @@ def test_volume_expand(self, mock_requests):
         url = '/v2/apps/{app_id}/volumes'.format(app_id=app_id)
         response = self.client.get(url)
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'name': 'myvolume1',
             'size': '1024G'
diff --git a/rootfs/api/tests/test_workflow_manager.py b/rootfs/api/tests/test_workflow_manager.py
index 5d655f86d..c2dd93ebc 100644
--- a/rootfs/api/tests/test_workflow_manager.py
+++ b/rootfs/api/tests/test_workflow_manager.py
@@ -40,18 +40,18 @@ def tearDown(self):
 
     def test_block(self, mock_requests):
         response = self.client.post(
-            '/v2/manager/{}/{}/block/'.format("users", 7),
+            '/v2/manager/{}/{}/block/'.format("apps", self.app_id),
             data={'remark': 'Arrears blockade'},
         )
         self.assertEqual(response.status_code, 201)
 
     def test_unblock(self, mock_requests):
         response = self.client.post(
-            '/v2/manager/{}/{}/block/'.format("users", 7),
+            '/v2/manager/{}/{}/block/'.format("apps", self.app_id),
             data={'remark': 'Arrears blockade'},
         )
         self.assertEqual(response.status_code, 201)
         response = self.client.delete(
-            '/v2/manager/{}/{}/unblock/'.format("users", 7),
+            '/v2/manager/{}/{}/unblock/'.format("apps", self.app_id),
         )
         self.assertEqual(response.status_code, 204)
diff --git a/rootfs/api/tests/test_workspace.py b/rootfs/api/tests/test_workspace.py
new file mode 100644
index 000000000..39476bd52
--- /dev/null
+++ b/rootfs/api/tests/test_workspace.py
@@ -0,0 +1,196 @@
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from unittest import mock
+
+from api.models.workspace import Workspace, WorkspaceMember, WorkspaceInvitation
+from api.tests import DryccTestCase
+
+User = get_user_model()
+
+
+class WorkspaceTest(DryccTestCase):
+    """Tests creation and management of workspaces"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        self.user2 = User.objects.get(username='autotest2')
+        self.token2 = self.get_or_create_token(self.user2)
+
+    def tearDown(self):
+        cache.clear()
+
+    def test_workspace_lifecycle(self):
+        """Test workspace create, list, retrieve, update, delete"""
+        # Create
+        response = self.client.post(
+            '/v2/workspaces', {'name': 'testworkspace', 'email': 'test@example.com'}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.data['name'], 'testworkspace')
+
+        # Verify admin member created
+        workspace = Workspace.objects.get(name='testworkspace')
+        self.assertTrue(workspace.has_member(self.user, role='admin'))
+
+        # List
+        response = self.client.get('/v2/workspaces')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 1)
+
+        # Retrieve
+        response = self.client.get('/v2/workspaces/testworkspace')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['name'], 'testworkspace')
+
+        # Update (personal workspace NOT allowed)
+        response = self.client.patch(
+            f'/v2/workspaces/{self.user.username}', {'email': 'new@example.com'}
+        )
+        self.assertEqual(response.status_code, 404, response.data)
+
+        # Update
+        response = self.client.patch('/v2/workspaces/testworkspace', {'email': 'new@example.com'})
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['email'], 'new@example.com')
+
+        # Delete
+        response = self.client.delete('/v2/workspaces/testworkspace')
+        self.assertEqual(response.status_code, 204, response.data)
+
+    def test_workspace_isolation(self):
+        # User 1 creates workspace
+        self.client.post('/v2/workspaces', {'name': 'testisolated', 'email': 'test@example.com'})
+
+        # User 2 tries to access User 1's workspace
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.get('/v2/workspaces/testisolated')
+        self.assertEqual(response.status_code, 404, response.data)  # Not found for user2
+
+
+class WorkspaceMemberTest(DryccTestCase):
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.user2 = User.objects.get(username='autotest2')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        # Create a workspace
+        self.client.post('/v2/workspaces', {'name': 'testmembers', 'email': 'test@example.com'})
+        self.workspace = Workspace.objects.get(name='testmembers')
+
+    def test_member_management(self):
+        # Add member via DB (since POST /members is not supported, users join via invitations)
+        WorkspaceMember.objects.create(user=self.user2, workspace=self.workspace, role='member')
+
+        # List members
+        response = self.client.get('/v2/workspaces/testmembers/members')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 2)
+
+        # Update member role (admin updating user2)
+        response = self.client.patch(
+            f'/v2/workspaces/testmembers/members/{self.user2.username}', {'role': 'viewer'}
+        )
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['role'], 'viewer')
+
+        # Non-admin user2 cannot update other members
+        token2 = self.get_or_create_token(self.user2)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + token2)
+        response = self.client.patch(
+            f'/v2/workspaces/testmembers/members/{self.user.username}', {'role': 'viewer'}
+        )
+        self.assertEqual(response.status_code, 403, response.data)
+
+        # Admin delete member
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        response = self.client.delete(f'/v2/workspaces/testmembers/members/{self.user2.username}')
+        self.assertEqual(response.status_code, 204, response.data)
+
+
+class WorkspaceInvitationTest(DryccTestCase):
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.user2 = User.objects.get(username='autotest2')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        self.client.post('/v2/workspaces', {'name': 'testinvite', 'email': 'test@example.com'})
+        self.workspace = Workspace.objects.get(name='testinvite')
+
+    @mock.patch('api.models.workspace.send_mail')
+    def test_invitation_lifecycle(self, mock_send_mail):
+        # Create invitation
+        response = self.client.post(
+            '/v2/workspaces/testinvite/invitations', {'email': self.user2.email}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        mock_send_mail.assert_called_once()
+
+        invitation = WorkspaceInvitation.objects.get(email=self.user2.email)
+
+        # List invitations
+        response = self.client.get('/v2/workspaces/testinvite/invitations')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 1)
+
+        # Accept invitation via API (JSON response)
+        response = self.client.get(f'/v2/workspaces/testinvite/invitations/{invitation.token}')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertIn('workspace_name', response.data)
+        self.assertEqual(response.data['workspace_name'], 'testinvite')
+        self.assertIn('user_exists', response.data)
+        self.assertIn('register_url', response.data)
+
+        # Verify user became a member
+        self.assertTrue(
+            WorkspaceMember.objects.filter(
+                workspace=self.workspace,
+                user=self.user2
+            ).exists()
+        )
+
+        # Test delete invitation
+        response = self.client.post(
+            '/v2/workspaces/testinvite/invitations', {'email': 'test-invite2@example.com'}
+        )
+        invitation2 = WorkspaceInvitation.objects.get(email='test-invite2@example.com')
+        response = self.client.delete(
+            f'/v2/workspaces/testinvite/invitations/{invitation2.token}'
+        )
+        self.assertEqual(response.status_code, 204, response.data)
+
+    @mock.patch('api.models.workspace.send_mail')
+    def test_invitation_accept_browser(self, mock_send_mail):
+        """Test accepting invitation via browser returns HTML"""
+        response = self.client.post(
+            '/v2/workspaces/testinvite/invitations', {'email': self.user2.email}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        invitation = WorkspaceInvitation.objects.get(email=self.user2.email)
+
+        # Accept invitation via browser (HTML response)
+        response = self.client.get(
+            f'/v2/workspaces/testinvite/invitations/{invitation.token}',
+            HTTP_ACCEPT='text/html',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response['Content-Type'], 'text/html; charset=utf-8')
+        self.assertIn(b'testinvite', response.content)
+
+        # Verify user became a member
+        self.assertTrue(
+            WorkspaceMember.objects.filter(
+                workspace=self.workspace,
+                user=self.user2
+            ).exists()
+        )
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 647b119fb..bf4baba6b 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -17,6 +17,27 @@
     re_path(r'auth/login/?$', views.AuthLoginView.as_view({"post": "login"})),
     re_path(r'auth/token/?$', views.AuthTokenView.as_view({"post": "token"})),
     re_path(r'auth/token/(?P<key>[-_\w]+)/?$', views.AuthTokenView.as_view({"get": "token"})),
+    # Workspaces management URLs (keep workspaces prefix, no user prefix)
+    re_path(r'^workspaces/?$',
+            views.WorkspaceViewSet.as_view({'get': 'list', 'post': 'create'}),
+            name='workspace_list'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/?$',
+            views.WorkspaceViewSet.as_view(
+                {'get': 'retrieve', 'patch': 'partial_update', 'delete': 'destroy'}),
+            name='workspace_detail'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/members/?$',
+            views.WorkspaceMemberViewSet.as_view({'get': 'list'}),
+            name='workspace_member_list'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/members/(?P<user>[-_\w]+)/?$',
+            views.WorkspaceMemberViewSet.as_view(
+                {'get': 'retrieve', 'patch': 'partial_update', 'delete': 'destroy'}),
+            name='workspace_member_detail'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/invitations/?$',
+            views.WorkspaceInvitationViewSet.as_view({'get': 'list', 'post': 'create'}),
+            name='workspace_invitation_list'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/invitations/(?P<uid>[-_\w]+)/?$',
+            views.WorkspaceInvitationViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'}),
+            name='workspace_invitation_detail'),
     # limits
     re_path(
         r'^limits/specs/?$',
@@ -33,7 +54,7 @@
         views.BuildViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
     re_path(
         r"^apps/(?P<id>{})/config/?$".format(settings.APP_URL_REGEX),
-        views.ConfigViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
+        views.ConfigViewSet.as_view({'get': 'retrieve', 'post': 'create', 'delete': 'destroy'})),
     re_path(
         r"^apps/(?P<id>{})/releases/v(?P<version>[0-9]+)/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'get': 'retrieve'})),
@@ -49,7 +70,7 @@
     # list/delete pods
     re_path(
         r"^apps/(?P<id>{})/pods/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list', 'delete': 'delete'})),
+        views.PodViewSet.as_view({'get': 'list', 'delete': 'destroy'})),
     # describe pod
     re_path(
         r"^apps/(?P<id>{})/pods/(?P<name>{})/describe/?$".format(
@@ -92,11 +113,7 @@
     re_path(
         r"^apps/(?P<id>{})/services/?$".format(settings.APP_URL_REGEX),
         views.ServiceViewSet.as_view({'post': 'create_or_update',
-                                     'get': 'list', 'delete': 'delete'})),
-    # application actions
-    re_path(
-        r"^apps/(?P<id>{})/run/?$".format(settings.APP_URL_REGEX),
-        views.AppViewSet.as_view({'post': 'run'})),
+                                     'get': 'list', 'delete': 'destroy'})),
     # application settings
     re_path(
         r"^apps/(?P<id>{})/settings/?$".format(settings.APP_URL_REGEX),
@@ -143,17 +160,18 @@
     re_path(
         r'^apps/(?P<id>{})/certs/(?P<name>{})/?$'.format(
             settings.APP_URL_REGEX, settings.NAME_REGEX),
-        views.CertificateViewSet.as_view({
-            'get': 'retrieve',
-            'delete': 'destroy'
-        })),
+        views.CertificateViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
     re_path(
         r'^apps/(?P<id>{})/certs/?$'.format(settings.APP_URL_REGEX),
         views.CertificateViewSet.as_view({'get': 'list', 'post': 'create'})),
+    # application actions
+    re_path(
+        r"^apps/(?P<id>{})/run/?$".format(settings.APP_URL_REGEX),
+        views.AppViewSet.as_view({'post': 'run'})),
     # apps base endpoint
     re_path(
         r"^apps/(?P<id>{})/?$".format(settings.APP_URL_REGEX),
-        views.AppViewSet.as_view({'get': 'retrieve', 'post': 'update', 'delete': 'destroy'})),
+        views.AppViewSet.as_view({'get': 'retrieve', 'patch': 'transfer', 'delete': 'destroy'})),
     re_path(
         r'^apps/?$',
         views.AppViewSet.as_view({'get': 'list', 'post': 'create'})),
@@ -183,18 +201,18 @@
     # authn / authz
     re_path(
         r'^auth/whoami/?$',
-        views.UserManagementViewSet.as_view({'get': 'list'})),
+        views.UserManagementViewSet.as_view({'get': 'whoami'})),
     # gateways
     re_path(
         r"^apps/(?P<id>{})/gateways/?$".format(settings.APP_URL_REGEX),
         views.GatewayViewSet.as_view(
-            {'post': 'create_or_update', 'get': 'list', 'delete': 'delete'})),
+            {'post': 'create_or_update', 'get': 'list', 'delete': 'destroy'})),
     # routes
     re_path(
         r"^apps/(?P<id>{})/routes/(?P<name>{})?/?$".format(
             settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view(
-            {'post': 'create', 'get': 'list', 'delete': 'delete'})),
+            {'post': 'create', 'get': 'list', 'delete': 'destroy'})),
     re_path(
         r"^apps/(?P<id>{})/routes/(?P<name>{})/attach/?$".format(
             settings.APP_URL_REGEX, settings.NAME_REGEX),
@@ -207,14 +225,6 @@
         r"^apps/(?P<id>{})/routes/(?P<name>{})/rules/?$".format(
             settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'get': 'get', 'put': 'set'})),
-    # users
-    re_path(r'^users/?$', views.UserView.as_view({'get': 'list'})),
-    re_path(
-        r'^users/(?P<username>[\w.@+-]+)/enable/?$',
-        views.UserView.as_view({'patch': 'enable'})),
-    re_path(
-        r'^users/(?P<username>[\w.@+-]+)/disable/?$',
-        views.UserView.as_view({'patch': 'disable'})),
     re_path(
         r'^apps/(?P<id>{})/metrics/?$'.format(settings.APP_URL_REGEX),
         views.MetricView.as_view({'get': 'metric'})),
@@ -228,13 +238,6 @@
     re_path(
         r'^manager/(?P<type>[\w.@+-]+)s/(?P<id>{})/unblock/?$'.format(settings.APP_URL_REGEX),
         views.WorkflowManagerViewset.as_view({'delete': 'unblock'})),
-    # user perms
-    re_path(
-        r"^apps/(?P<id>{})/perms/?$".format(settings.APP_URL_REGEX),
-        views.AppPermViewSet.as_view({'get': 'list', 'post': 'create'})),
-    re_path(
-        r"^apps/(?P<id>{})/perms/(?P<username>[\w.@+-]+)/?$".format(settings.APP_URL_REGEX),
-        views.AppPermViewSet.as_view({'put': 'update', 'delete': 'destroy'})),
     # quickwit
     re_path(
         r'^quickwit/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$', views.QuickwitProxyView.as_view()),
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index e3c40b9d4..d2aa1d519 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -240,6 +240,11 @@ def validate_json(value, schema, raise_exception=jsonschema.ValidationError):
     return value
 
 
+def get_local_host(request):
+    uri = request.build_absolute_uri()
+    return uri[0:uri.find(request.path)]
+
+
 class CacheLock(object):
 
     def __init__(self, key):
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index ef30259e9..131889d5a 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -9,6 +9,7 @@
 import time
 import zlib
 import random
+import secrets
 import aiohttp
 import requests
 import warnings
@@ -20,22 +21,20 @@
 from django.http import Http404, HttpResponse
 from django.conf import settings
 from django.contrib.auth import get_user_model
-from django.shortcuts import get_object_or_404, redirect
-from guardian.shortcuts import assign_perm, get_objects_for_user, \
-    get_users_with_perms, get_user_perms, remove_perm
+from django.shortcuts import get_object_or_404, redirect, render
 from django.views.generic import View
 from django.utils.decorators import method_decorator
 from django.views.decorators.cache import cache_page
 from django.views.decorators.vary import vary_on_headers
-from rest_framework import renderers, status, filters
+from rest_framework import status, filters
 from rest_framework.permissions import IsAuthenticated, AllowAny
 from rest_framework.response import Response
-from rest_framework.viewsets import GenericViewSet
-from rest_framework.exceptions import PermissionDenied
+from rest_framework.viewsets import GenericViewSet, ModelViewSet
+from rest_framework.exceptions import ValidationError
+from rest_framework.renderers import JSONRenderer, TemplateHTMLRenderer
 
 from api import monitor, models, permissions, serializers, viewsets, authentication, __version__
-from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner, \
-    delete_pod, scale_resources
+from api.tasks import scale_app, restart_app, mount_app, delete_pod, scale_resources
 from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
@@ -168,14 +167,8 @@ def token(self, request, *args, **kwargs):
 class UserManagementViewSet(GenericViewSet):
     serializer_class = serializers.UserSerializer
 
-    def get_queryset(self):
-        return User.objects.filter(pk=self.request.user.pk)
-
-    def get_object(self):
-        return self.get_queryset()[0]
-
-    def list(self, request, **kwargs):
-        user = self.get_object()
+    def whoami(self, request, **kwargs):
+        user = get_object_or_404(User, pk=self.request.user.pk)
         serializer = self.get_serializer(user, many=False)
         return Response(serializer.data)
 
@@ -253,10 +246,234 @@ def handle(self, request,  **kwargs):
         })
 
 
-class TokenViewSet(viewsets.OwnerViewSet):
+class WorkspaceViewSet(ModelViewSet):
+    """
+    ViewSet for Workspace model.
+    """
+    lookup_field = 'name'
+    serializer_class = serializers.WorkspaceSerializer
+    permission_classes = [IsAuthenticated]
+
+    def get_queryset(self):
+        return models.workspace.Workspace.objects.filter(
+            workspacemember__user=self.request.user
+        ).distinct()
+
+    def perform_create(self, serializer):
+        workspace = serializer.save()
+        models.workspace.WorkspaceMember.objects.create(
+            user=self.request.user, workspace=workspace, role='admin'
+        )
+
+    def get_object(self):
+        """Override to get workspace by name instead of pk"""
+        return get_object_or_404(self.get_queryset(), name=self.kwargs['name'])
+
+    def partial_update(self, request, *args, **kwargs):
+        """Only admins can update workspaces"""
+        workspace = self.get_object()
+        if not workspace.has_member(request.user, role='admin'):
+            return Response(
+                {"detail": "Only workspace admins can update workspaces"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        return super().partial_update(request, *args, **kwargs)
+
+    def destroy(self, request, *args, **kwargs):
+        """Only admins can delete workspaces"""
+        workspace = self.get_object()
+        if not workspace.has_member(request.user, role='admin'):
+            return Response(
+                {"detail": "Only workspace admins can delete workspaces"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        if models.workspace.WorkspaceMember.objects.filter(workspace=workspace).count() > 1:
+            return Response(
+                {"detail": "Cannot delete workspace with more than one member"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        return super().destroy(request, *args, **kwargs)
+
+
+class WorkspaceMemberViewSet(ModelViewSet):
+    """
+    ViewSet for WorkspaceMember model.
+    """
+    serializer_class = serializers.WorkspaceMemberSerializer
+    permission_classes = [IsAuthenticated]
+
+    def get_queryset(self):
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        # Check if user has access to this workspace
+        if workspace.has_member(self.request.user):
+            return models.workspace.WorkspaceMember.objects.filter(workspace=workspace)
+        return models.workspace.WorkspaceMember.objects.none()
+
+    def get_object(self):
+        """Override to get member by username and workspace name"""
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        return get_object_or_404(
+            models.workspace.WorkspaceMember,
+            workspace=workspace, user__username=self.kwargs['user']
+        )
+
+    def partial_update(self, request, *args, **kwargs):
+        """Update a member. Admins can update any member (role and alerts).
+        Non-admins can only update their own alerts field."""
+        member = self.get_object()
+        is_admin = member.workspace.has_member(request.user, role='admin')
+
+        # Check if workspace has only one member
+        member_count = models.workspace.WorkspaceMember.objects.filter(
+            workspace=member.workspace
+        ).count()
+        is_only_member = member_count == 1
+
+        # Only member cannot modify role
+        if is_only_member and 'role' in request.data:
+            return Response(
+                {"detail": "Cannot modify role: workspace only has one member"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+
+        # Non-admin users restrictions
+        if not is_admin:
+            # Cannot update other members
+            if request.user != member.user:
+                return Response(
+                    {"detail": "Only workspace admins can update other members"},
+                    status=status.HTTP_403_FORBIDDEN
+                )
+            # Cannot modify own role
+            if 'role' in request.data:
+                return Response(
+                    {"detail": "Cannot modify your own role"},
+                    status=status.HTTP_403_FORBIDDEN
+                )
+
+        return super().partial_update(request, *args, **kwargs)
 
+    def destroy(self, request, *args, **kwargs):
+        """Delete a member. Admins can delete any member.
+        Non-admins can only delete themselves (leave workspace)."""
+        member = self.get_object()
+        is_admin = member.workspace.has_member(request.user, role='admin')
+
+        # Check if workspace has only one member
+        member_count = models.workspace.WorkspaceMember.objects.filter(
+            workspace=member.workspace
+        ).count()
+        is_only_member = member_count == 1
+
+        # Only member cannot delete self
+        if is_only_member and request.user == member.user:
+            return Response(
+                {"detail": "Cannot delete: workspace only has one member"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+
+        # Non-admin can delete self
+        if request.user == member.user:
+            return super().destroy(request, *args, **kwargs)
+
+        # Admin can delete any member
+        if is_admin:
+            return super().destroy(request, *args, **kwargs)
+
+        # Other cases forbidden
+        return Response(
+            {"detail": "Only workspace admins can remove other members"},
+            status=status.HTTP_403_FORBIDDEN
+        )
+
+
+class WorkspaceInvitationViewSet(ModelViewSet):
+    """
+    ViewSet for WorkspaceInvitation model.
+    """
+    serializer_class = serializers.WorkspaceInvitationSerializer
+
+    def get_permissions(self):
+        """
+        Allow anyone to accept an invitation.
+        Only authenticated users can create or list invitations.
+        """
+        if self.action == 'retrieve':
+            return [AllowAny()]
+        return super().get_permissions()
+
+    def get_renderers(self):
+        if self.action == 'retrieve':
+            return [JSONRenderer(), TemplateHTMLRenderer()]
+        return super().get_renderers()
+
+    def get_queryset(self):
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        if workspace.has_member(self.request.user):
+            return models.workspace.WorkspaceInvitation.objects.filter(
+                workspace=workspace, accepted=False)
+        return models.workspace.WorkspaceInvitation.objects.none()
+
+    def get_object(self):
+        """Override to get invitation by uid and workspace name"""
+        return get_object_or_404(
+            models.workspace.WorkspaceInvitation,
+            workspace=get_object_or_404(models.workspace.Workspace, name=self.kwargs['name']),
+            token=self.kwargs['uid'],
+            accepted=False,
+        )
+
+    def retrieve(self, request, *args, **kwargs):
+        instance = self.get_object()
+        instance.accept()
+        user_exists = User.objects.filter(email=instance.email).exists()
+        data = {
+            'workspace_name': instance.workspace.name,
+            'user_exists': user_exists,
+            'register_url': settings.DRYCC_REGISTER_URL,
+        }
+        if isinstance(request.accepted_renderer, TemplateHTMLRenderer):
+            return render(request, 'workspace/workspace_invitation_accept.html', data)
+        return Response(data)
+
+    def perform_create(self, serializer):
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        if not workspace.has_member(self.request.user, role='admin'):
+            raise ValidationError("Only workspace admins can create invitations")
+        email = serializer.validated_data['email']
+        user = User.objects.filter(email=email).first()
+        if user and workspace.has_member(user):
+            raise ValidationError("User is already a member of the workspace")
+        invitation = models.workspace.WorkspaceInvitation.objects.filter(
+            email=email, workspace=workspace, accepted=False
+        ).first()
+        if not invitation:
+            models.workspace.WorkspaceInvitation.objects.filter(
+                email=email, workspace=workspace, accepted=True
+            ).delete()
+            invitation = serializer.save(
+                token=secrets.token_hex(64), inviter=self.request.user, workspace=workspace)
+        if settings.EMAIL_HOST:
+            invitation.send_email(self.request)
+        else:
+            invitation.accept()
+
+    def destroy(self, request, *args, **kwargs):
+        """Only admins can revoke invitations"""
+        invitation = self.get_object()
+        if not invitation.workspace.has_member(request.user, role='admin'):
+            return Response(
+                {"detail": "Only workspace admins can revoke invitations"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        return super().destroy(request, *args, **kwargs)
+
+
+class TokenViewSet(viewsets.OwnerViewSet):
+    """
+    A viewset for interacting with Token objects.
+    """
     serializer_class = serializers.TokenSerializer
-    permission_classes = [IsAuthenticated, permissions.IsOwner]
 
     def get_queryset(self):
         return models.base.Token.objects.filter(owner=self.request.user)
@@ -268,19 +485,7 @@ def destroy(self, *args, **kwargs):
         return response
 
 
-class BaseDryccViewSet(viewsets.OwnerViewSet):
-    """
-    A generic ViewSet for objects related to Drycc.
-
-    To use it, at minimum you'll need to provide the `serializer_class` attribute and
-    the `model` attribute shortcut.
-    """
-    lookup_field = 'id'
-    permission_classes = [IsAuthenticated, permissions.IsObjectUser]
-    renderer_classes = [renderers.JSONRenderer]
-
-
-class AppFilterViewSet(BaseDryccViewSet):
+class AppFilterViewSet(viewsets.BaseAppViewSet):
     """A viewset for objects which are attached to an application."""
 
     def get_app(self):
@@ -314,34 +519,13 @@ def get_object(self):
         return getattr(release, self.model.__name__.lower())
 
 
-class AppViewSet(BaseDryccViewSet):
+class AppViewSet(viewsets.BaseAppViewSet):
     """A viewset for interacting with App objects."""
     model = models.app.App
     filter_backends = [filters.SearchFilter]
     search_fields = ['^id', ]
     serializer_class = serializers.AppSerializer
 
-    def get_queryset(self, *args, **kwargs):
-        return self.model.objects.all(*args, **kwargs)
-
-    def list(self, request, *args, **kwargs):
-        """
-        HACK: Instead of filtering by the queryset, we limit the queryset to list only the apps
-        which are owned by the user as well as any apps they have been given permission to
-        interact with.
-        """
-        queryset = super().get_queryset(**kwargs) | \
-            get_objects_for_user(
-                self.request.user, f'api.{models.app.VIEW_APP_PERMISSION.codename}')
-        instance = self.filter_queryset(queryset)
-        page = self.paginate_queryset(instance)
-        if page is not None:
-            serializer = self.get_serializer(page, many=True)
-            return self.get_paginated_response(serializer.data)
-
-        serializer = self.get_serializer(instance, many=True)
-        return Response(serializer.data)
-
     def run(self, request, **kwargs):
         app = self.get_object()
         ptype = request.data.get('ptype', 'run')
@@ -363,23 +547,21 @@ def run(self, request, **kwargs):
                 timeout=timeout, expires=expires)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    @transaction.atomic
-    def create(self, request, *args, **kwargs):
-        return super().create(request, *args, **kwargs)
+    def perform_create(self, serializer):
+        workspace = serializer.validated_data['workspace']
+        self.check_object_permissions(self.request, workspace)
+        serializer.save()
 
-    @transaction.atomic
-    def update(self, request, **kwargs):
+    def transfer(self, request, **kwargs):
         app = self.get_object()
-        old_owner = app.owner
-
-        if request.data.get('owner'):
-            if self.request.user != app.owner and not self.request.user.is_superuser:
-                return Response(status=status.HTTP_403_FORBIDDEN)
-            new_owner = get_object_or_404(User, username=request.data['owner'])
-            # ensure all downstream objects that are owned by this user and are part of this app
-            # is also updated
-            downstream_model_owner.delay(app, old_owner, new_owner)
-        return Response(status=status.HTTP_200_OK)
+        if not app.workspace.has_member(request.user, role='admin'):
+            raise DryccException("you must be an admin of the current workspace")
+        workspace = request.data.get('workspace', '')
+        if not workspace:
+            raise DryccException("workspace is required")
+        app.workspace = get_object_or_404(models.workspace.Workspace, name=workspace)
+        app.save()
+        return Response(status=status.HTTP_204_NO_CONTENT)
 
     @transaction.atomic
     def destroy(self, request, *args, **kwargs):
@@ -391,16 +573,16 @@ class BuildViewSet(ReleasableViewSet):
     model = models.build.Build
     serializer_class = serializers.BuildSerializer
 
-    def post_save(self, build):
+    def perform_create(self, serializer):
+        build = serializer.save()
         for ptype in build.ptypes:
             image = build.get_image(ptype)
             if is_loopback(image):
                 raise DryccException("image must not use the loopback address")
         build.create_release(self.request.user)
-        super().post_save(build)
 
 
-class LimitSpecViewSet(BaseDryccViewSet):
+class LimitSpecViewSet(ModelViewSet):
     """A viewset for interacting with Limit objects."""
     model = models.limit.LimitSpec
     serializer_class = serializers.LimitSpecSerializer
@@ -414,7 +596,7 @@ def get_queryset(self, **kwargs):
         return self.model.objects.filter(q)
 
 
-class LimitPlanViewSet(BaseDryccViewSet):
+class LimitPlanViewSet(ModelViewSet):
     """A viewset for interacting with Limit objects."""
     model = models.limit.LimitPlan
     serializer_class = serializers.LimitPlanSerializer
@@ -445,7 +627,7 @@ def create(self, request, **kwargs):
         if self.request.query_params.get('merge', 'true').lower() == 'true':
             return super().create(request, **kwargs)
         values = self.get_serializer().validate_values(request.data.get('values'))
-        config = self.model(app=self.get_app(), owner=self.request.user, values=values)
+        config = self.model(app=self.get_app(), values=values)
         old_config = config.previous()
         if old_config and old_config.values:
             replace_ptypes = {v['ptype'] for v in config.values if 'ptype' in v}
@@ -457,13 +639,19 @@ def create(self, request, **kwargs):
         headers = self.get_success_headers(data)
         return Response(data, status=status.HTTP_201_CREATED, headers=headers)
 
-    def delete(self, request, **kwargs):
-        values_refs = self.get_serializer().validate_values_refs(request.data.get('values_refs'))
+    def perform_create(self, serializer):
+        config = serializer.save()
+        self.post_save(config)
+
+    def destroy(self, request, **kwargs):
+        values_refs = self.get_serializer().validate_values_refs(
+            request.data.get('values_refs', {}))
         if not values_refs or not values_refs.values():
             raise DryccException("ptype or groups is required")
 
-        config = self.model(app=self.get_app(), owner=self.request.user, values_refs={})
-        old_values_refs = config.previous().values_refs.copy()
+        config = self.model(app=self.get_app(), values_refs={})
+        previous = config.previous()
+        old_values_refs = previous.values_refs.copy() if previous else {}
         for ptype, old_groups in old_values_refs.items():
             groups_to_delete = values_refs.get(ptype, [])
             for group in old_groups:
@@ -473,7 +661,6 @@ def delete(self, request, **kwargs):
                     elif group not in config.values_refs[ptype]:
                         config.values_refs[ptype].append(group)
         config.save(ignore_update_fields=["values_refs"])
-
         self.post_save(config)
         return Response(status=status.HTTP_200_OK)
 
@@ -511,7 +698,7 @@ def describe(self, *args, **kwargs):
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
 
-    def delete(self, request, **kwargs):
+    def destroy(self, request, **kwargs):
         pod_names = request.data.get("pod_ids")
         pod_names = pod_names.split(",")
         for pod_name in set(pod_names):
@@ -640,13 +827,13 @@ def create_or_update(self, request, **kwargs):
                     return Response(status=status.HTTP_400_BAD_REQUEST, data={"detail": "port is occupied"})  # noqa
             http_status = status.HTTP_204_NO_CONTENT
         else:
-            service = self.model(owner=app.owner, app=app, ptype=ptype)
+            service = self.model(app=app, ptype=ptype)
             http_status = status.HTTP_201_CREATED
         service.add_port(port, protocol, target_port)
         service.save()
         return Response(status=http_status)
 
-    def delete(self, request, **kwargs):
+    def destroy(self, request, **kwargs):
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data.get('protocol'))
         ptype = self.get_serializer().validate_ptype(
@@ -691,10 +878,10 @@ def detach(self, request, *args, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class KeyViewSet(BaseDryccViewSet):
+class KeyViewSet(viewsets.OwnerViewSet):
     """A viewset for interacting with Key objects."""
+    lookup_field = 'id'
     model = models.key.Key
-    permission_classes = [IsAuthenticated, permissions.IsOwner]
     serializer_class = serializers.KeySerializer
 
 
@@ -758,7 +945,7 @@ def events(self, request, **kwargs):
         return Response({'results': results, 'count': len(results)})
 
 
-class BaseHookViewSet(BaseDryccViewSet):
+class BaseHookViewSet(viewsets.BaseAppViewSet):
     permission_classes = [permissions.IsServiceToken]
 
 
@@ -770,31 +957,22 @@ class KeyHookViewSet(BaseHookViewSet):
     def public_key(self, request, *args, **kwargs):
         fingerprint = kwargs['fingerprint'].strip()
         key = get_object_or_404(models.key.Key, fingerprint=fingerprint)
-
-        queryset = models.app.App.objects.all() | \
-            get_objects_for_user(
-                self.request.user, f'api.{models.app.VIEW_APP_PERMISSION.codename}')
-        items = self.filter_queryset(queryset)
-
-        apps = []
-        for item in items:
-            apps.append(item.id)
-
+        queryset = models.app.App.objects.filter(
+            workspace__workspacemember__user=self.request.user
+        ).distinct()
         data = {
             'username': key.owner.username,
-            'apps': apps
+            'apps': [item.id for item in self.filter_queryset(queryset)],
         }
 
         return Response(data, status=status.HTTP_200_OK)
 
     def app(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
-        usernames = [u.id for u in get_users_with_perms(app)
-                     if u.has_perm(f"api.{models.app.VIEW_APP_PERMISSION.codename}", app)]
-
+        usernames = app.workspace.workspacemember_set.values_list('user__username', flat=True)
         data = {}
         result = models.key.Key.objects \
-                       .filter(owner__in=usernames) \
+                       .filter(owner__username__in=usernames) \
                        .values('owner__username', 'public', 'fingerprint') \
                        .order_by('created')
         for info in result:
@@ -813,9 +991,8 @@ def users(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
         request.user = get_object_or_404(User, username=kwargs['username'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request.user, app, request.method)
-        if not has_permission:
-            return Response(message, status=status.HTTP_403_FORBIDDEN)
+        if not permissions.IsAppUser().has_object_permission(request, self, app):
+            return Response(status=status.HTTP_403_FORBIDDEN)
 
         data = {request.user.username: []}
         keys = models.key.Key.objects \
@@ -843,9 +1020,8 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         self.user = request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(self.user, app, request.method)
-        if not has_permission:
-            return Response(message, status=status.HTTP_403_FORBIDDEN)
+        if not permissions.IsAppUser().has_object_permission(request, self, app):
+            return Response(status=status.HTTP_403_FORBIDDEN)
         request.data['app'] = app
         request.data['owner'] = self.user
         super().create(request, *args, **kwargs)
@@ -857,7 +1033,8 @@ def create(self, request, *args, **kwargs):
         }
         return Response(response, status=status.HTTP_200_OK)
 
-    def post_save(self, build):
+    def perform_create(self, serializer):
+        build = serializer.save()
         build.create_release(self.user)
 
 
@@ -870,90 +1047,13 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request.user, app, request.method)
-        if not has_permission:
-            return Response(message, status=status.HTTP_403_FORBIDDEN)
+        if not permissions.IsAppUser().has_object_permission(request, self, app):
+            return Response(status=status.HTTP_403_FORBIDDEN)
         config = models.release.Release.latest(app).config
         serializer = self.get_serializer(config)
         return Response(serializer.data, status=status.HTTP_200_OK)
 
 
-class AppPermViewSet(AppFilterViewSet):
-    """RESTful views for sharing apps with collaborators."""
-
-    def get_app(self, request):
-        app = get_object_or_404(models.app.App, id=self.kwargs['id'])
-        if not permissions.IsOwnerOrAdmin().has_object_permission(request, self, app):
-            raise PermissionDenied()
-        return app
-
-    def list(self, request, **kwargs):
-        app = self.get_app(request)
-        results = [
-            {
-                "app": app.id,
-                "username": user.username,
-                "permissions": [
-                    models.app.app_permission_registry.get(codename).shortname
-                    for codename in get_user_perms(user, app)
-                ],
-            }
-            for user in get_users_with_perms(app)
-        ]
-        # fake out pagination for now
-        pagination = {'results': results, 'count': len(results)}
-        return Response(data=pagination)
-
-    def create(self, request, **kwargs):
-        app = self.get_app(request)
-        username = request.data.get('username')
-        shortnames = set([perm for perm in request.data.get("permissions", "").split(",") if perm])
-        all_shortnames = models.app.app_permission_registry.shortnames
-        if not shortnames or not shortnames.issubset(all_shortnames):
-            msg = "The permissions field is required and has a value range of: {}".format(
-                ",".join(all_shortnames)
-            )
-            return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
-        user = get_object_or_404(User, username=username)
-        for shortname in shortnames:
-            permission = models.app.app_permission_registry.get(shortname)
-            if permission:
-                assign_perm(permission.codename, user, app)
-        app.log("User {} was granted access to {}".format(user, app))
-        return Response(status=status.HTTP_201_CREATED)
-
-    def update(self, request, **kwargs):
-        app = self.get_app(request)
-        user = get_object_or_404(User, username=kwargs['username'])
-        shortnames = set([perm for perm in request.data.get("permissions", "").split(",") if perm])
-        all_shortnames = models.app.app_permission_registry.shortnames
-        if not shortnames or not shortnames.issubset(all_shortnames):
-            msg = "The permissions field is required and has a value range of: {}".format(
-                ",".join(all_shortnames)
-            )
-            return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
-        for shortname in shortnames.symmetric_difference([
-                models.app.app_permission_registry.get(codename).shortname
-                for codename in get_user_perms(user, app)]):
-            permission = models.app.app_permission_registry.get(shortname)
-            if permission:
-                if shortname in shortnames:
-                    assign_perm(permission.codename, user, app)
-                else:
-                    remove_perm(permission.codename, user, app)
-        app.log("User {} was revoked access to {}".format(user, app))
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-    def destroy(self, request, **kwargs):
-        app = self.get_app(request)
-        username = kwargs['username']
-        user = get_object_or_404(User, username=username)
-        for codename in get_user_perms(user, app):
-            remove_perm(codename, user, app)
-        app.log("User {} was revoked access to {}".format(user, app))
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
 class AppVolumesViewSet(AppFilterViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
@@ -1101,14 +1201,14 @@ def create_or_update(self, request, *args, **kwargs):
         protocol = self.get_serializer().validate_protocol(request.data['protocol'])
         gateway = app.gateway_set.filter(name=name).first()
         if not gateway:
-            gateway = self.model(app=app, owner=app.owner, name=name)
+            gateway = self.model(app=app, name=name)
         added, msg = gateway.add(port, protocol)
         if not added:
             return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
         gateway.save()
         return Response(status=status.HTTP_201_CREATED)
 
-    def delete(self, request, **kwargs):
+    def destroy(self, request, **kwargs):
         app = self.get_app()
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data['protocol'])
@@ -1171,40 +1271,14 @@ def detach(self, request, *args, **kwargs):
         route.save()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def delete(self, request, *args, **kwargs):
+    def destroy(self, request, *args, **kwargs):
         app = self.get_app()
         route = get_object_or_404(self.model, app=app, name=kwargs['name'])
         route.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class UserView(BaseDryccViewSet):
-    """A Viewset for interacting with User objects."""
-    model = User
-    serializer_class = serializers.UserSerializer
-    permission_classes = [permissions.IsAdmin]
-
-    def get_queryset(self):
-        return self.model.objects.exclude(username='AnonymousUser')
-
-    def enable(self, request, **kwargs):
-        if request.user.username == kwargs['username']:
-            return Response(status=status.HTTP_423_LOCKED)
-        user = get_object_or_404(self.model, username=kwargs['username'])
-        user.is_active = True
-        user.save(update_fields=['is_active', ])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-    def disable(self, request, **kwargs):
-        if request.user.username == kwargs['username']:
-            return Response(status=status.HTTP_423_LOCKED)
-        user = get_object_or_404(self.model, username=kwargs['username'])
-        user.is_active = False
-        user.save(update_fields=['is_active', ])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-class MetricView(BaseDryccViewSet):
+class MetricView(viewsets.BaseAppViewSet):
     """Getting monitoring indicators from monitor database"""
 
     def _get_app(self):
@@ -1229,7 +1303,7 @@ class MetricsProxyView(View):
         re.compile(r'^(?:# (?:HELP|TYPE) )([a-zA-Z_][a-zA-Z0-9_:.-]*)').match)
     match_data = staticmethod(
         re.compile(r'^([a-zA-Z_][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)').match)
-    default_cache_value = (-1, -1)
+    default_cache_value = (None, -1)
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -1254,17 +1328,17 @@ async def sample(self, name, labels_str, value):
         app_id = labels.get("namespace", labels.get(settings.DRYCC_METRICS_CONFIG[name][0]))
         if not app_id:
             return None
-        account_id, timeout = self.cache.get(app_id, self.default_cache_value)
-        if (account_id < 0 and timeout < 0) or time.time() > timeout:
+        workspace_id, timeout = self.cache.get(app_id, self.default_cache_value)
+        if workspace_id is None or time.time() > timeout:
             if app := await models.app.App.objects.filter(id=app_id).afirst():
-                account_id = app.owner_id
+                workspace_id = app.workspace_id
             else:
-                account_id = -1
-            self.cache[app_id] = (account_id, time.time() + random.randint(600, 1200))
-        if account_id < 0:
+                workspace_id = None
+            self.cache[app_id] = (workspace_id, time.time() + random.randint(600, 1200))
+        if workspace_id is None:
             return None
         project_id = zlib.crc32(app_id.encode("utf-8"))
-        labels.update({'vm_account_id': account_id, 'vm_project_id': project_id})
+        labels.update({'vm_account_id': workspace_id, 'vm_project_id': project_id})
         return "%s{%s} %s\n" % (name, ",".join([f'{k}="{v}"' for k, v in labels.items()]), value)
 
     async def get(self, request):
diff --git a/rootfs/api/viewsets.py b/rootfs/api/viewsets.py
index 4e972fdd5..0871f5ba7 100644
--- a/rootfs/api/viewsets.py
+++ b/rootfs/api/viewsets.py
@@ -1,4 +1,5 @@
-from rest_framework import viewsets
+from django.core.exceptions import ImproperlyConfigured
+from rest_framework import viewsets, renderers
 from rest_framework.permissions import IsAuthenticated
 
 from api import permissions
@@ -17,12 +18,25 @@ def get_queryset(self):
         return self.model.objects.filter(owner=self.request.user)
 
     def perform_create(self, serializer):
-        obj = serializer.save(owner=self.request.user)
-        self.post_save(obj)
-
-    def post_save(self, obj):
-        """
-        A post_save hook for performing actions after the object has been pushed to the database.
-        Leave it up to child classes to implement.
-        """
-        pass
+        serializer.save(owner=self.request.user)
+
+
+class BaseAppViewSet(viewsets.ModelViewSet):
+    """
+    A ViewSet for the Workspace model, which filters workspaces by membership and role.
+    """
+    lookup_field = 'id'
+    permission_classes = [IsAuthenticated, permissions.IsAppUser]
+    renderer_classes = [renderers.JSONRenderer]
+
+    def get_queryset(self):
+        # Prefer direct workspace relation, then support app->workspace chain.
+        if hasattr(self.model, 'workspace'):
+            return self.model.objects.filter(
+                workspace__workspacemember__user=self.request.user).distinct()
+        elif hasattr(self.model, 'app'):
+            return self.model.objects.filter(
+                app__workspace__workspacemember__user=self.request.user).distinct()
+        raise ImproperlyConfigured(
+            f"{self.__class__.__name__} requires a model with a 'workspace' or 'app' field."
+        )
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index c8ca06e04..0c80ff269 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -5,7 +5,6 @@ django==5.2.9
 channels==4.3.1
 aiohttp==3.13.3
 django-cors-headers==4.9.0
-django-guardian==3.2.0
 djangorestframework==3.16.1
 docker==7.1.0
 gunicorn==23.0.0
@@ -15,12 +14,9 @@ idna==3.11
 jsonschema==4.25.1
 morph==0.1.5
 packaging==25.0
-pyasn1==0.6.2
 psycopg[binary]==3.2.12
-pyOpenSSL==25.3.0
-ndg-httpsclient==0.5.1
 pytz==2025.2
-requests==2.32.5
+requests==2.33.0
 requests-toolbelt==1.0.0
 celery[redis]==5.5.3
 hiredis==3.3.0
