diff --git a/.gitignore b/.gitignore
index 1eb305349..2d9ec853d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ var
 sdist
 develop-eggs
 .installed.cfg
+.venv
 lib
 lib64
 
@@ -44,4 +45,4 @@ vendor/
 # generated bintray scripts during ci
 client/_scripts/ci/bintray-ci.json
 .idea
-.vscode/
\ No newline at end of file
+.vscode/
diff --git a/.woodpecker/build-linux.yml b/.woodpecker/build-linux.yml
index 8cdd984e5..802f8754d 100644
--- a/.woodpecker/build-linux.yml
+++ b/.woodpecker/build-linux.yml
@@ -1,30 +1,35 @@
 matrix:
   platform:
-    - linux/amd64
-    - linux/arm64
-
-platform: ${platform}
+  - linux/amd64
+  - linux/arm64
 
 labels:
   type: exec
+  platform: ${platform}
 
-pipeline:
+steps:
 - name: publish-linux
   image: bash
   commands:
-  - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_ARCH)
+  - export VERSION=$([ -z $CI_COMMIT_TAG ] && echo latest || echo $CI_COMMIT_TAG)-$(sed 's#/#-#g' <<< $CI_SYSTEM_PLATFORM)
   - echo $CONTAINER_PASSWORD | podman login $DRYCC_REGISTRY --username $CONTAINER_USERNAME --password-stdin > /dev/null 2>&1
   - make podman-build podman-immutable-push
-  secrets:
-  - codename
-  - dev_registry
-  - drycc_registry
-  - container_username
-  - container_password
+  environment:
+    CODENAME:
+      from_secret: codename
+    DEV_REGISTRY:
+      from_secret: dev_registry
+    DRYCC_REGISTRY:
+      from_secret: drycc_registry
+    CONTAINER_USERNAME:
+      from_secret: container_username
+    CONTAINER_PASSWORD:
+      from_secret: container_password
   when:
     event:
     - push
     - tag
+    - cron
 
 depends_on:
 - test-linux
\ No newline at end of file
diff --git a/.woodpecker/chart.yaml b/.woodpecker/chart.yaml
index f27bcf551..63eb318c5 100644
--- a/.woodpecker/chart.yaml
+++ b/.woodpecker/chart.yaml
@@ -1,30 +1,34 @@
-platform: linux/amd64
-
 labels:
   type: exec
+  platform: linux/amd64
 
-pipeline:
+steps:
 - name: generate-chart
-  type: local
   image: bash
   commands:
   - export VERSION=$(sed 's#v##' <<< $CI_COMMIT_TAG)
   - export IMAGE_TAG=$([ ! -z $CI_COMMIT_TAG ] && echo \"$VERSION\" || echo \"canary\")
-  - export APP_VERSION=$([ ! -z $CI_COMMIT_TAG ] && echo $VERSION || echo 1.0.0)
+  - export APP_VERSION=$([ -z $CI_COMMIT_TAG ] && echo $CI_COMMIT_SHA || echo $VERSION)
   - export CHART_VERSION=$([ -z $CI_COMMIT_TAG ] && echo 1.0.0 || echo $VERSION)
+  - _scripts/util/update_filer.sh
   - sed -i "s/imageTag:\ \"canary\"/imageTag:\ $IMAGE_TAG/g" charts/$${CI_REPO_NAME}/values.yaml
   - helm package -u charts/$${CI_REPO_NAME} --version $CHART_VERSION --app-version $APP_VERSION
   - echo $CONTAINER_PASSWORD | helm registry login $DRYCC_REGISTRY -u $CONTAINER_USERNAME --password-stdin
   - helm push $${CI_REPO_NAME}-$CHART_VERSION.tgz oci://$DRYCC_REGISTRY/$([ -z $CI_COMMIT_TAG ] && echo charts-testing || echo charts)
-  secrets:
-  - dev_registry
-  - drycc_registry
-  - container_username
-  - container_password
+  environment:
+    DEV_REGISTRY:
+      from_secret: dev_registry
+    DRYCC_REGISTRY:
+      from_secret: drycc_registry
+    CONTAINER_USERNAME:
+      from_secret: container_username
+    CONTAINER_PASSWORD:
+      from_secret: container_password
   when:
     event:
     - push
     - tag
+    - cron
 
 depends_on:
 - manifest
diff --git a/.woodpecker/manifest.yml b/.woodpecker/manifest.yml
index 3dced546b..88ded6677 100644
--- a/.woodpecker/manifest.yml
+++ b/.woodpecker/manifest.yml
@@ -1,20 +1,21 @@
-platform: linux/amd64
-
 labels:
   type: exec
+  platform: linux/amd64
 
-pipeline:
+steps:
 - name: generate-manifest
   image: bash
   commands:
   - sed -i "s/{{project}}/$${CI_REPO_NAME}/g" .woodpecker/manifest.tmpl
   - sed -i "s/registry.drycc.cc/$${DRYCC_REGISTRY}/g" .woodpecker/manifest.tmpl
-  secrets:
-  - drycc_registry
+  environment:
+    DRYCC_REGISTRY:
+      from_secret: drycc_registry
   when:
     event:
     - tag
     - push
+    - cron
 
 - name: publish-manifest
   image: bash
@@ -27,13 +28,16 @@ pipeline:
     -v $(pwd):$(pwd)
     -w $(pwd)
     plugins/manifest
-  secrets:
-  - container_username
-  - container_password
+  environment:
+    CONTAINER_USERNAME:
+      from_secret: container_username
+    CONTAINER_PASSWORD:
+      from_secret: container_password
   when:
     event:
     - tag
     - push
+    - cron
 
 depends_on:
 - build-linux
diff --git a/.woodpecker/test-linux.yml b/.woodpecker/test-linux.yml
index 63e57e3e1..f22a5f0fe 100644
--- a/.woodpecker/test-linux.yml
+++ b/.woodpecker/test-linux.yml
@@ -3,21 +3,24 @@ matrix:
   - linux/amd64
   - linux/arm64
 
-platform: ${platform}
-
 labels:
   type: exec
+  platform: ${platform}
 
-pipeline:
+steps:
 - name: test-linux
   image: bash
   commands:
   - make test podman-build-test upload-coverage
-  secrets:
-  - codename
-  - dev_registry
-  - codecov_token
+  environment:
+    CODENAME:
+      from_secret: codename
+    DEV_REGISTRY:
+      from_secret: dev_registry
+    CODECOV_TOKEN:
+      from_secret: codecov_token
   when:
     event:
     - push
     - tag
+    - cron
diff --git a/Makefile b/Makefile
index 62c6b6ac7..da92b03f1 100644
--- a/Makefile
+++ b/Makefile
@@ -63,6 +63,6 @@ test-integration:
 
 upload-coverage:
 	$(eval CI_ENV := $(shell curl -s https://codecov.io/env | bash))
-	podman run --rm ${CI_ENV} -v ${CURDIR}:/test -w /test/rootfs ${IMAGE}.test /test/rootfs/bin/upload-coverage
+	podman run --rm ${CI_ENV} -v ${CURDIR}:/test -w /test/rootfs -e CODECOV_TOKEN=${CODECOV_TOKEN} ${IMAGE}.test /test/rootfs/bin/upload-coverage
 
 .PHONY: check-kubectl check-podman build podman-build podman-build-test deploy clean commit-hook full-clean test test-style test-unit test-functional test-integration upload-coverage
diff --git a/_scripts/util/update_filer.sh b/_scripts/util/update_filer.sh
new file mode 100755
index 000000000..885b28a58
--- /dev/null
+++ b/_scripts/util/update_filer.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+if [ -n "${CI_COMMIT_TAG}" ]; then
+    version=$(curl -Ls https://github.com/drycc/filer/releases|grep /drycc/filer/releases/tag/ | sed -E 's/.*\/drycc\/filer\/releases\/tag\/(v[0-9\.]{1,}(-rc.[0-9]{1,})?)".*/\1/g' | head -1)
+    sed -i "s|registry.drycc.cc/drycc/filer:canary|registry.drycc.cc/drycc/filer:${version#v}|g" charts/controller/values.yaml
+fi
\ No newline at end of file
diff --git a/charts/controller/files/limit-plans.json b/charts/controller/files/limit-plans.json
new file mode 100644
index 000000000..2b9a7fe29
--- /dev/null
+++ b/charts/controller/files/limit-plans.json
@@ -0,0 +1,982 @@
+[
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "1Gi"
+      },
+      "memory": 1,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10100,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "0.5Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m1"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "2Gi"
+      },
+      "memory": 2,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10200,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "1Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m2"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "4Gi"
+      },
+      "memory": 4,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10300,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "2Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m4"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 1,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 1,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10400,
+      "requests": {
+        "cpu": 0.25,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c1m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "2Gi"
+      },
+      "memory": 2,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10500,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "1Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m2"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "4Gi"
+      },
+      "memory": 4,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10600,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "2Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m4"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10700,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 2,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 2,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10800,
+      "requests": {
+        "cpu": 0.5,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c2m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "4Gi"
+      },
+      "memory": 4,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 10900,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "2Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m4"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11000,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11100,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 4,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 4,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11200,
+      "requests": {
+        "cpu": 1,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c4m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "8Gi"
+      },
+      "memory": 8,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11300,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "4Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m8"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11400,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11500,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 8,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 8,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11600,
+      "requests": {
+        "cpu": 2,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c8m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "16Gi"
+      },
+      "memory": 16,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11700,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "8Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m16"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11800,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 11900,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 16,
+      "created": "2025-09-16T05:05:37.136Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 16,
+        "ephemeral-storage": "4Gi",
+        "memory": "128Gi"
+      },
+      "memory": 128,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12000,
+      "requests": {
+        "cpu": 4,
+        "ephemeral-storage": "2Gi",
+        "memory": "64Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.136Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c16m128"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "32Gi"
+      },
+      "memory": 32,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12100,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "16Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m32"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12200,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "128Gi"
+      },
+      "memory": 128,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12300,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "64Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m128"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 32,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 32,
+        "ephemeral-storage": "4Gi",
+        "memory": "256Gi"
+      },
+      "memory": 256,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12400,
+      "requests": {
+        "cpu": 8,
+        "ephemeral-storage": "2Gi",
+        "memory": "128Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c32m256"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "64Gi"
+      },
+      "memory": 64,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12500,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "32Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m64"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "128Gi"
+      },
+      "memory": 128,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12600,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "64Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m128"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "256Gi"
+      },
+      "memory": 256,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12700,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "128Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m256"
+  },
+  {
+    "fields": {
+      "annotations": {
+        "kubernetes.io/egress-bandwidth": "100M",
+        "kubernetes.io/ingress-bandwidth": "100M"
+      },
+      "container_security_context": {},
+      "cpu": 64,
+      "created": "2025-09-16T05:05:37.137Z",
+      "disabled": false,
+      "features": {
+        "gpu": 1,
+        "network": 1
+      },
+      "limits": {
+        "cpu": 64,
+        "ephemeral-storage": "4Gi",
+        "memory": "512Gi"
+      },
+      "memory": 512,
+      "node_selector": {},
+      "pod_security_context": {},
+      "priority": 12800,
+      "requests": {
+        "cpu": 16,
+        "ephemeral-storage": "2Gi",
+        "memory": "256Gi"
+      },
+      "runtime_class_name": "",
+      "spec_id": "std1",
+      "updated": "2025-09-16T05:05:37.137Z"
+    },
+    "model": "api.limitplan",
+    "pk": "std1.large.c64m512"
+  }
+]
diff --git a/charts/controller/files/limit-specs.json b/charts/controller/files/limit-specs.json
new file mode 100644
index 000000000..732a70e2d
--- /dev/null
+++ b/charts/controller/files/limit-specs.json
@@ -0,0 +1,38 @@
+[
+  {
+    "fields": {
+      "cpu": {
+        "boost": "3700MHZ",
+        "clock": "3100MHZ",
+        "cores": 32,
+        "name": "Universal CPU",
+        "threads": 64
+      },
+      "created": "2025-09-16T05:05:37.135Z",
+      "disabled": false,
+      "features": {
+        "gpu": {
+          "cores": 128,
+          "memory": {
+            "size": "shared",
+            "type": "shared"
+          },
+          "name": "Integrated GPU",
+          "rops": 1,
+          "tmus": 1
+        },
+        "network": "10G"
+      },
+      "keywords": [
+        "unknown"
+      ],
+      "memory": {
+        "size": "64GB",
+        "type": "DDR4-ECC"
+      },
+      "updated": "2025-09-16T05:05:37.135Z"
+    },
+    "model": "api.limitspec",
+    "pk": "std1"
+  }
+]
diff --git a/charts/controller/files/metrics.json b/charts/controller/files/metrics.json
new file mode 100644
index 000000000..e620eeec2
--- /dev/null
+++ b/charts/controller/files/metrics.json
@@ -0,0 +1,466 @@
+{
+  "container_cpu_load_average_10s": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_system_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_usage_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_user_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_cfs_periods_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_cfs_throttled_periods_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_cpu_cfs_throttled_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_inodes_free": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_usage_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_inodes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_io_current": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_io_time_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_io_time_weighted_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_reads_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_read_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_reads_merged_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_reads_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_sector_reads_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_sector_writes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_writes_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_write_seconds_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_writes_merged_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_fs_writes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_blkio_device_usage_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_failures_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_failcnt": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_cache": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_mapped_file": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_max_usage_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_rss": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_swap": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_usage_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_memory_working_set_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_errors_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_packets_dropped_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_receive_packets_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_bytes_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_errors_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_packets_dropped_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_network_transmit_packets_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_oom_events_total": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_processes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_sockets": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_file_descriptors": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_threads": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_threads_max": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_ulimits_soft": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_cpu_quota": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_cpu_period": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_cpu_shares": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_memory_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_memory_reservation_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_spec_memory_swap_limit_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_start_time_seconds": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_tasks_state": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_last_seen": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_accelerator_memory_used_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_accelerator_memory_total_bytes": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "container_accelerator_duty_cycle": [
+    "instance",
+    "namespace",
+    "pod",
+    "container"
+  ],
+  "kube_pod_ips": [
+    "ip",
+    "ip_family",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_running": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_ready": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_terminated": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_waiting": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_container_status_restarts_total": [
+    "container",
+    "namespace",
+    "node",
+    "pod"
+  ],
+  "kube_pod_spec_volumes_persistentvolumeclaims_info": [
+    "namespace",
+    "pod",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_used_bytes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_available_bytes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_capacity_bytes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_inodes": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_inodes_free": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "kubelet_volume_stats_inodes_used": [
+    "namespace",
+    "persistentvolumeclaim"
+  ],
+  "istio_requests_total": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "response_code",
+    "connection_security_policy",
+    "source_workload"
+  ],
+  "istio_request_duration_milliseconds_bucket": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "connection_security_policy",
+    "source_workload",
+    "le"
+  ],
+  "istio_request_bytes_bucket": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "connection_security_policy",
+    "source_workload",
+    "le"
+  ],
+  "istio_response_bytes_bucket": [
+    "source_workload_namespace",
+    "reporter",
+    "destination_service",
+    "connection_security_policy",
+    "source_workload",
+    "le"
+  ]
+}
\ No newline at end of file
diff --git a/charts/controller/files/network-usage-template.promql b/charts/controller/files/network-usage-template.promql
new file mode 100644
index 000000000..07a608de1
--- /dev/null
+++ b/charts/controller/files/network-usage-template.promql
@@ -0,0 +1,9 @@
+label_replace(
+  sum(increase(istio_request_bytes_sum{namespace=~"{{namespaces}}"}[{{duration}}])) by (namespace,pod),
+  "direction", "rx", "", ""
+)
+or
+label_replace(
+  sum(increase(istio_response_bytes_sum{namespace=~"{{namespaces}}"}[{{duration}}])) by (namespace,pod),
+  "direction", "tx", "", ""
+)
diff --git a/charts/controller/files/reserved-name-patterns.txt b/charts/controller/files/reserved-name-patterns.txt
new file mode 100644
index 000000000..1b6ca23bc
--- /dev/null
+++ b/charts/controller/files/reserved-name-patterns.txt
@@ -0,0 +1,13 @@
+^backup$
+^catalog$
+^cert-manager$
+^default$
+^drycc(?:-[\w-]+)?$
+^istio(?:-[\w-]+)?$
+^kube(?:-[\w-]+)?$
+^longhorn-system$
+^metallb$
+^mount-s3$
+^topolvm$
+^rook-ceph$
+^personal$
\ No newline at end of file
diff --git a/charts/controller/files/volume-claim-template.json b/charts/controller/files/volume-claim-template.json
new file mode 100644
index 000000000..be8133bfa
--- /dev/null
+++ b/charts/controller/files/volume-claim-template.json
@@ -0,0 +1,61 @@
+{% if type == 'csi' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "usage"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "{{storage_class}}",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeMode": "Filesystem"
+  }
+}
+{% elif type == 'nfs' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% elif type == 'oss' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% endif %}
diff --git a/charts/controller/files/volume-template.json b/charts/controller/files/volume-template.json
new file mode 100644
index 000000000..b4a54f132
--- /dev/null
+++ b/charts/controller/files/volume-template.json
@@ -0,0 +1,55 @@
+{% if type == 'nfs' %}
+{
+  "spec": {
+    "capacity": {
+      "storage": "65535Gi"
+    },
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "claimRef": {
+      "namespace": "{{namespace}}",
+      "name": "{{volume_claim_name}}"
+    },
+    "nfs": {
+      "server": "{{server}}",
+      "path": "{{path}}"
+    },
+    "mountOptions": [
+      "nfsvers=4.2"
+    ]
+  }
+}
+{% elif type == 'oss' %}
+{
+  "spec": {
+    "capacity": {
+      "storage": "65535Gi"
+    },
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "claimRef": {
+      "namespace": "{{namespace}}",
+      "name": "{{volume_claim_name}}"
+    },
+    "mountOptions": [
+      "allow-delete",
+      {% if path_style %}
+      "force-path-style",
+      {% endif %}
+      "aws-access-key-id {{access_key}}",
+      "aws-secret-access-key {{secret_key}}",
+      "endpoint-url {{server}}"
+    ],
+    "csi": {
+      "driver": "s3.csi.aws.com",
+      "volumeHandle": "{{volume_handle}}",
+      "volumeAttributes": {
+        "bucketName": "{{bucket}}"
+      }
+    }
+  }
+}
+{% endif %}
diff --git a/charts/controller/files/volume-usage-template.promql b/charts/controller/files/volume-usage-template.promql
new file mode 100644
index 000000000..fec7f1913
--- /dev/null
+++ b/charts/controller/files/volume-usage-template.promql
@@ -0,0 +1,12 @@
+max by(namespace, persistentvolumeclaim, storageclass) (
+  kube_persistentvolumeclaim_resource_requests_storage_bytes{
+    namespace=~"{{namespaces}}"
+  }
+  * on(namespace, persistentvolumeclaim) group_left(storageclass)
+  kube_persistentvolumeclaim_info{namespace=~"{{namespaces}}"}
+  * on(namespace, persistentvolumeclaim) group_left()
+  kube_persistentvolumeclaim_annotations{
+    namespace=~"{{namespaces}}",
+    annotation_billing_drycc_cc_type="usage"
+  }
+)
diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
index 26f304817..80902a4d2 100644
--- a/charts/controller/templates/_helpers.tpl
+++ b/charts/controller/templates/_helpers.tpl
@@ -1,29 +1,28 @@
 {{/* Generate controller deployment envs */}}
 {{- define "controller.envs" }}
 env:
+- name: VERSION
+  value: {{ .Chart.AppVersion }}
 - name: REGISTRATION_MODE
   value: {{ .Values.registrationMode }}
-# Environmental variable value for $GATEWAY_CLASS
-- name: "DRYCC_GATEWAY_CLASS"
-  value: "{{ .Values.global.gatewayClass }}"
 - name: "K8S_API_VERIFY_TLS"
   value: "{{ .Values.k8sApiVerifyTls }}"
 - name: "DRYCC_REGISTRY_LOCATION"
-  value: "{{ .Values.global.registryLocation }}"
+  value: {{ ternary "on-cluster" "off-cluster" .Values.registry.enabled }}
 - name: "DRYCC_REGISTRY_SECRET_PREFIX"
-  value: "{{ .Values.global.registrySecretPrefix }}"
+  value: "{{ .Values.registrySecretPrefix }}"
 - name: "IMAGE_PULL_POLICY"
   value: "{{ .Values.appImagePullPolicy }}"
-- name: "KUBERNETES_CLUSTER_DOMAIN"
-  value: "{{ .Values.global.clusterDomain }}"
+- name: "DRYCC_FILER_IMAGE"
+  value: "{{ (tpl .Values.filerImage .) }}"
+- name: "DRYCC_FILER_IMAGE_PULL_POLICY"
+  value: "{{ (tpl .Values.filerImagePullPolicy .) }}"
+- name: "DRYCC_APP_GATEWAY_CLASS"
+  value: "{{ .Values.appGatewayClass }}"
 {{- if (.Values.appStorageClass) }}
 - name: "DRYCC_APP_STORAGE_CLASS"
   value: "{{ (tpl .Values.appStorageClass .) }}"
 {{- end }}
-{{- if (.Values.appRuntimeClass) }}
-- name: "DRYCC_APP_RUNTIME_CLASS"
-  value: "{{ .Values.appRuntimeClass }}"
-{{- end }}
 {{- if (.Values.appDNSPolicy) }}
 - name: "DRYCC_APP_DNS_POLICY"
   value: "{{ .Values.appDNSPolicy }}"
@@ -50,11 +49,26 @@ env:
     secretKeyRef:
       name: controller-creds
       key: django-secret-key
-- name: DRYCC_BUILDER_KEY
+- name: DRYCC_SERVICE_KEY
+  valueFrom:
+    secretKeyRef:
+      name: controller-creds
+      key: service-key
+{{- if (.Values.valkeyUrl) }}
+- name: DRYCC_VALKEY_URL
+  valueFrom:
+    secretKeyRef:
+      name: controller-creds
+      key: valkey-url
+{{- else if .Values.valkey.enabled }}
+- name: VALKEY_PASSWORD
   valueFrom:
     secretKeyRef:
-      name: builder-key-auth
-      key: builder-key
+      name: valkey-creds
+      key: password
+- name: DRYCC_VALKEY_URL
+  value: "redis://:$(VALKEY_PASSWORD)@drycc-valkey:16379/0"
+{{- end }}
 {{- if (.Values.databaseReplicaUrl) }}
 - name: DRYCC_DATABASE_REPLICA_URL
   valueFrom:
@@ -68,7 +82,7 @@ env:
     secretKeyRef:
       name: controller-creds
       key: database-url
-{{- else if eq .Values.global.databaseLocation "on-cluster"  }}
+{{- else if .Values.database.enabled }}
 - name: DRYCC_PG_USER
   valueFrom:
     secretKeyRef:
@@ -80,29 +94,9 @@ env:
       name: database-creds
       key: password
 - name: DRYCC_DATABASE_URL
-  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/controller"
+  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database:5432/controller"
 - name: DRYCC_DATABASE_REPLICA_URL
-  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/controller"
-{{- end }}
-{{- if (.Values.databaseMonitorUrl) }}
-- name: DRYCC_DATABASE_MONITOR_URL
-  valueFrom:
-    secretKeyRef:
-      name: controller-creds
-      key: database-monitor-url
-{{- else if eq .Values.global.timeseriesLocation "on-cluster"  }}
-- name: DRYCC_TS_USER
-  valueFrom:
-    secretKeyRef:
-      name: timeseries-creds
-      key: user
-- name: DRYCC_TS_PASSWORD
-  valueFrom:
-    secretKeyRef:
-      name: timeseries-creds
-      key: password
-- name: DRYCC_DATABASE_MONITOR_URL
-  value: "postgres://$(DRYCC_TS_USER):$(DRYCC_TS_PASSWORD)@drycc-timeseries-replica.{{.Release.Namespace}}.svc.{{.Values.global.clusterDomain}}:5432/monitor"
+  value: "postgres://$(DRYCC_PG_USER):$(DRYCC_PG_PASSWORD)@drycc-database-replica:5432/controller"
 {{- end }}
 {{- if (.Values.workflowManagerUrl) }}
 - name: WORKFLOW_MANAGER_URL
@@ -112,61 +106,29 @@ env:
 - name: WORKFLOW_MANAGER_SECRET_KEY
   value: "{{ .Values.workflowManagerSecretKey }}"
 {{- end }}
-- name: WORKFLOW_NAMESPACE
+- name: POD_IP
   valueFrom:
     fieldRef:
-      fieldPath: metadata.namespace
-- name: DRYCC_REDIS_ADDRS
-  valueFrom:
-    secretKeyRef:
-      name: redis-creds
-      key: addrs
-- name: DRYCC_REDIS_PASSWORD
+      fieldPath: status.podIP
+- name: POD_NAME
   valueFrom:
-    secretKeyRef:
-      name: redis-creds
-      key: password
-{{- if eq .Values.global.rabbitmqLocation "off-cluster" }}
-- name: "DRYCC_PROMETHEUS_URL"
-  valueFrom:
-    secretKeyRef:
-      name: prometheus-creds
-      key: url
-{{- else }}
-- name: "DRYCC_PROMETHEUS_USERNAME"
-  valueFrom:
-    secretKeyRef:
-      name: prometheus-creds
-      key: username
-- name: "DRYCC_PROMETHEUS_PASSWORD"
+    fieldRef:
+      fieldPath: metadata.name
+- name: WORKFLOW_NAMESPACE
   valueFrom:
-    secretKeyRef:
-      name: prometheus-creds
-      key: password
-- name: "DRYCC_PROMETHEUS_URL"
-  value: "http://$(DRYCC_PROMETHEUS_USERNAME):$(DRYCC_PROMETHEUS_PASSWORD)@drycc-prometheus.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:9090"
-{{- end }}
-{{- if (.Values.rabbitmqUrl) }}
-- name: DRYCC_RABBITMQ_URL
+    fieldRef:
+      fieldPath: metadata.namespace
+{{- if (.Values.victoriametricsUrl) }}
+- name: "DRYCC_VICTORIAMETRICS_URL"
   valueFrom:
     secretKeyRef:
       name: controller-creds
-      key: rabbitmq-url
-{{- else if eq .Values.global.rabbitmqLocation "on-cluster" }}
-- name: "DRYCC_RABBITMQ_USERNAME"
-  valueFrom:
-    secretKeyRef:
-      name: rabbitmq-creds
-      key: username
-- name: "DRYCC_RABBITMQ_PASSWORD"
-  valueFrom:
-    secretKeyRef:
-      name: rabbitmq-creds
-      key: password
-- name: "DRYCC_RABBITMQ_URL"
-  value: "amqp://$(DRYCC_RABBITMQ_USERNAME):$(DRYCC_RABBITMQ_PASSWORD)@drycc-rabbitmq.{{$.Release.Namespace}}.svc.{{$.Values.global.clusterDomain}}:5672/drycc"
+      key: victoriametrics-url
+{{- else if .Values.victoriametrics.enabled }}
+- name: "DRYCC_VICTORIAMETRICS_URL"
+  value: "http://drycc-victoriametrics-vmselect:8481/select/multitenant/prometheus"
 {{- end }}
-{{- if eq .Values.global.passportLocation "on-cluster"}}
+{{- if .Values.passport.enabled }}
 - name: "DRYCC_PASSPORT_URL"
 {{- if .Values.global.certManagerEnabled }}
   value: https://drycc-passport.{{ .Values.global.platformDomain }}
@@ -200,22 +162,38 @@ env:
       name: controller-creds
       key: passport-secret
 {{- end }}
+- name: QUICKWIT_INDEXER_URL
+  value: http://drycc-quickwit-indexer:7280
+- name: QUICKWIT_SEARCHER_URL
+  value: http://drycc-quickwit-searcher:7280
+- name: QUICKWIT_LOG_INDEX_PREFIX
+  value: {{ .Values.quickwit.logIndexPrefix }}
 {{- range $key, $value := .Values.environment }}
 - name: {{ $key }}
   value: {{ $value | quote }}
 {{- end }}
 {{- end }}
 
-{{/* Generate controller deployment limits */}}
-{{- define "controller.limits" -}}
-{{- if or (.Values.limitsCpu) (.Values.limitsMemory) }}
-resources:
-  limits:
-{{- if (.Values.limitsCpu) }}
-    cpu: {{.Values.limitsCpu}}
+{{- define "controller-job.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_DATABASE_ROUTERS
+  value: api.routers.DefaultReplicaRouter
 {{- end }}
-{{- if (.Values.limitsMemory) }}
-    memory: {{.Values.limitsMemory}}
+
+{{- define "controller-api.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_CONTROLLER_RUNNER
+  value: api
 {{- end }}
+
+{{- define "controller-metric.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_CONTROLLER_RUNNER
+  value: metric
 {{- end }}
+
+{{- define "controller-mutate.envs" }}
+{{- include "controller.envs" . }}
+- name: DRYCC_CONTROLLER_RUNNER
+  value: mutate
 {{- end }}
diff --git a/charts/controller/templates/controller-api-deployment.yaml b/charts/controller/templates/api/controller-api-deployment.yaml
similarity index 63%
rename from charts/controller/templates/controller-api-deployment.yaml
rename to charts/controller/templates/api/controller-api-deployment.yaml
index 271e41524..c8fdf45f6 100644
--- a/charts/controller/templates/controller-api-deployment.yaml
+++ b/charts/controller/templates/api/controller-api-deployment.yaml
@@ -1,13 +1,13 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: drycc-controller
+  name: drycc-controller-api
   labels:
     heritage: drycc
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
 spec:
-  replicas: {{ .Values.apiReplicas }}
+  replicas: {{ .Values.api.replicas }}
   strategy:
     rollingUpdate:
       maxSurge: 1
@@ -15,11 +15,13 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: drycc-controller-api
+      app: drycc-controller
+      component: drycc-controller-api
   template:
     metadata:
       labels: {{- include "common.labels.standard" . | nindent 8 }}
-        app: drycc-controller-api
+        app: drycc-controller
+        component: drycc-controller-api
     spec:
       affinity:
         podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.api.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.api.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
@@ -27,21 +29,24 @@ spec:
         nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.api.nodeAffinityPreset.type "key" .Values.api.nodeAffinityPreset.key "values" .Values.api.nodeAffinityPreset.values ) | nindent 10 }}
       serviceAccount: drycc-controller
       initContainers:
-      - name: drycc-controller-init
+      - name: drycc-controller-api-init
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
         imagePullPolicy: {{.Values.imagePullPolicy}}
         args:
         - netcat
         - -v
         - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-        - -a
-        - $(DRYCC_REDIS_ADDRS)
-        {{- include "controller.envs" . | indent 8 }}
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
+        {{- include "controller-api.envs" . | indent 8 }}
       containers:
-      - name: drycc-controller
+      - name: drycc-controller-api
         image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
         imagePullPolicy: {{.Values.imagePullPolicy}}
+        {{- if .Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
+        args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if not .Values.diagnosticMode.enabled }}
         startupProbe:
           tcpSocket:
             port: 8000
@@ -63,8 +68,20 @@ spec:
           initialDelaySeconds: 30
           timeoutSeconds: 10
           periodSeconds: 5
+        {{- end }}
         ports:
           - containerPort: 8000
             name: http
-        {{- include "controller.limits" . | indent 8 }}
-        {{- include "controller.envs" . | indent 8 }}
+        {{- with index .Values "api" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- include "controller-api.envs" . | indent 8 }}
+        volumeMounts:
+        - name: controller-config
+          readOnly: false
+          mountPath: /etc/controller
+      volumes:
+      - name: controller-config
+        configMap:
+          name: controller-config
\ No newline at end of file
diff --git a/charts/controller/templates/controller-api-service.yaml b/charts/controller/templates/api/controller-api-service.yaml
similarity index 69%
rename from charts/controller/templates/controller-api-service.yaml
rename to charts/controller/templates/api/controller-api-service.yaml
index 82208897a..189f8d36e 100644
--- a/charts/controller/templates/controller-api-service.yaml
+++ b/charts/controller/templates/api/controller-api-service.yaml
@@ -7,11 +7,14 @@ metadata:
     {{- toYaml . | nindent 4 }}
     {{- end }}
   labels:
+    app: drycc-controller
     heritage: drycc
+    component: drycc-controller-api
 spec:
   ports:
     - name: http
       port: 80
       targetPort: 8000
   selector:
-    app: drycc-controller-api
+    app: drycc-controller
+    component: drycc-controller-api
diff --git a/charts/controller/templates/controller-celery-deloyment.yaml b/charts/controller/templates/celery/controller-celery-deloyment.yaml
similarity index 58%
rename from charts/controller/templates/controller-celery-deloyment.yaml
rename to charts/controller/templates/celery/controller-celery-deloyment.yaml
index 8840b3eca..ea9211efe 100644
--- a/charts/controller/templates/controller-celery-deloyment.yaml
+++ b/charts/controller/templates/celery/controller-celery-deloyment.yaml
@@ -7,7 +7,7 @@ metadata:
   annotations:
     component.drycc.cc/version: {{ .Values.imageTag }}
 spec:
-  replicas: {{ .Values.celeryReplicas }}
+  replicas: {{ .Values.celery.replicas }}
   strategy:
     rollingUpdate:
       maxSurge: 1
@@ -15,11 +15,13 @@ spec:
     type: RollingUpdate
   selector:
     matchLabels:
-      app: drycc-controller-celery
+      app: drycc-controller
+      component: drycc-controller-celery
   template:
     metadata:
       labels: {{- include "common.labels.standard" . | nindent 8 }}
-        app: drycc-controller-celery
+        app: drycc-controller
+        component: drycc-controller-celery
     spec:
       affinity:
         podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.celery.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.celery.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
@@ -34,17 +36,33 @@ spec:
           - netcat
           - -v
           - -u
-          - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-          - -a
-          - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
+          - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
         {{- include "controller.envs" . | indent 8 }}
       containers:
-      - name: drycc-controller-celery
+      {{- range $key := (list "low" "middle" "high") }}
+      - name: drycc-controller-celery-{{$key}}
         image: {{$.Values.imageRegistry}}/{{$.Values.imageOrg}}/controller:{{$.Values.imageTag}}
         imagePullPolicy: {{$.Values.imagePullPolicy}}
+        {{- if $.Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.command "context" $) | nindent 10 }}
+        args: {{- include "common.tplvalues.render" (dict "value" $.Values.diagnosticMode.args "context" $) | nindent 10 }}
+        {{- else }}
         args:
           - /bin/bash
           - -c
-          - celery -A api worker --autoscale=32,1 --loglevel=WARNING
-        {{- include "controller.limits" $ | indent 8 }}
+          - celery --app api worker -n {{uuidv4}}@%h --queues controller.{{$key}} --autoscale=32,1 --loglevel=WARNING
+        {{- end }}
+        {{- with index $.Values "celery" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
         {{- include "controller.envs" $ | indent 8 }}
+        volumeMounts:
+        - name: controller-config
+          readOnly: false
+          mountPath: /etc/controller
+      {{- end }}
+      volumes:
+      - name: controller-config
+        configMap:
+          name: controller-config
diff --git a/charts/controller/templates/controller-clusterrole.yaml b/charts/controller/templates/controller-clusterrole.yaml
index d16752740..3809127e3 100644
--- a/charts/controller/templates/controller-clusterrole.yaml
+++ b/charts/controller/templates/controller-clusterrole.yaml
@@ -1,4 +1,3 @@
-{{- if (.Values.global.rbac) -}}
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -16,7 +15,7 @@ rules:
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "list"]
-- apiGroups: [""]
+- apiGroups: ["events.k8s.io"]
   resources: ["events"]
   verbs: ["list", "create"]
 - apiGroups: [""]
@@ -52,24 +51,24 @@ rules:
 - apiGroups: ["extensions", "apps"]
   resources: ["deployments/scale", "replicasets/scale"]
   verbs: ["get", "update"]
+- apiGroups: ["apps"]
+  resources: ["statefulsets"]
+  verbs: ["get", "list", "patch"]
+- apiGroups: ["extensions", "apps"]
+  resources: ["daemonsets"]
+  verbs: ["get", "list", "patch"]
 - apiGroups: ["extensions", "autoscaling"]
   resources: ["horizontalpodautoscalers"]
   verbs: ["get", "list", "create", "update", "delete"]
-- apiGroups: ["extensions"]
-  resources: ["ingresses"]
-  verbs: ["get", "list", "watch", "create", "update", "delete"]
 - apiGroups: ["cert-manager.io"]
-  resources: ["certificates", "issuers"]
+  resources: ["certificates", "certificaterequest", "issuers"]
   verbs: ["get", "list", "watch", "create", "update", "delete"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingresses"]
-  verbs: ["get", "list", "create", "delete", "deletecollection", "patch", "update"]
-- apiGroups: ["networking.k8s.io"]
-  resources: ["ingressclasses"]
-  verbs: ["get"]
 - apiGroups: ["apps"]
   resources: ["replicasets"]
   verbs: ["get", "list", "watch", "delete"]
+- apiGroups: [""]
+  resources: ["persistentvolumes"]
+  verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
 - apiGroups: [""]
   resources: ["persistentvolumeclaims"]
   verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
@@ -80,7 +79,8 @@ rules:
   resources: ["serviceinstances", "servicebindings"]
   verbs: ["get", "list", "watch", "create", "delete", "patch", "update"]
 - apiGroups: ["gateway.networking.k8s.io"]
-  resources: ["gateways", "httproutes", "grocroutes", "tcproutes", "udproutes", "tlsroutes"]
+  resources: ["gateways", "httproutes", "grpcroutes", "tcproutes", "udproutes", "tlsroutes"]
   verbs: ["get", "patch", "list", "create", "update", "delete"]
-{{- end -}}
-
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["create"]
diff --git a/charts/controller/templates/controller-clusterrolebinding.yaml b/charts/controller/templates/controller-clusterrolebinding.yaml
index bf6517e94..23ee8d184 100644
--- a/charts/controller/templates/controller-clusterrolebinding.yaml
+++ b/charts/controller/templates/controller-clusterrolebinding.yaml
@@ -1,4 +1,3 @@
-{{- if (.Values.global.rbac) -}}
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
@@ -14,4 +13,3 @@ subjects:
 - kind: ServiceAccount
   name: drycc-controller
   namespace: {{ .Release.Namespace }}
-{{- end -}}
diff --git a/charts/controller/templates/controller-configmap.yaml b/charts/controller/templates/controller-configmap.yaml
new file mode 100644
index 000000000..18493da9a
--- /dev/null
+++ b/charts/controller/templates/controller-configmap.yaml
@@ -0,0 +1,55 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: controller-config
+  labels:
+    heritage: drycc
+data:
+  metrics.json: |
+    {{- if .Values.config.metrics }}
+    {{- (tpl .Values.config.metrics $) | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/metrics.json" | nindent 4 }}
+    {{- end }}
+  limit-specs.json: |
+    {{- if .Values.config.limitSpecs }}
+    {{- (tpl .Values.config.limitSpecs $)  | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/limit-specs.json" | nindent 4 }}
+    {{- end }}
+  limit-plans.json: |
+    {{- if .Values.config.limitPlans }}
+    {{- (tpl .Values.config.limitPlans $)  | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/limit-plans.json" | nindent 4 }}
+    {{- end }}
+  volume-template.json: |
+    {{- if .Values.config.volumeTemplate }}
+    {{- (tpl .Values.config.volumeTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/volume-template.json" | nindent 4 }}
+    {{- end }}
+  reserved-name-patterns.txt: |-
+    {{- if .Values.config.reservedNames }}
+    {{- (tpl .Values.config.reservedNames $)  | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/reserved-name-patterns.txt" | nindent 4 }}
+    {{- end }}
+  volume-claim-template.json: |
+    {{- if .Values.config.volumeClaimTemplate }}
+    {{- (tpl .Values.config.volumeClaimTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{-  .Files.Get "files/volume-claim-template.json" | nindent 4 }}
+    {{- end }}
+  volume-usage-template.promql: |
+    {{- if .Values.config.volumeUsageTemplate }}
+    {{- (tpl .Values.config.volumeUsageTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/volume-usage-template.promql" | nindent 4 }}
+    {{- end }}
+  network-usage-template.promql: |
+    {{- if .Values.config.networkUsageTemplate }}
+    {{- (tpl .Values.config.networkUsageTemplate $)  | nindent 4 }}
+    {{- else}}
+    {{- .Files.Get "files/network-usage-template.promql" | nindent 4 }}
+    {{- end }}
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
index 52f4b1325..6146c9300 100644
--- a/charts/controller/templates/controller-cronjob-daily.yaml
+++ b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -16,8 +16,9 @@ spec:
     spec:
       template:
         metadata:
-          labels:
-            app: drycc-controller-conjob
+          labels: {{- include "common.labels.standard" . | nindent 12 }}
+            app: drycc-controller
+            component: drycc-controller-conjob-daily
         spec:
           restartPolicy: OnFailure
           serviceAccount: drycc-controller
@@ -29,48 +30,46 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-              - -a
-              - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
-            {{- include "controller.envs" . | indent 12 }}
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
+            {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
             name: drycc-controller-clearsocial
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
             args:
             - /bin/bash
             - -c
             - python -u /workspace/manage.py clearsocial
+            {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.imagePullPolicy}}
             name: drycc-controller-load-db-state-to-k8s
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
             args:
               - /bin/bash
               - -c
               - python /workspace/manage.py load_db_state_to_k8s
+            {{- end }}
             {{- include "controller.envs" . | indent 12 }}
-          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-            imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-apps
-            args:
-              - /bin/bash
-              - -c
-              - python -u /workspace/manage.py measure_apps
-            {{- include "controller.envs" . | indent 12 }}
-          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-            imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-resources
-            args:
-            - /bin/bash
-            - -c
-            - python -u /workspace/manage.py measure_resources
-            {{- include "controller.envs" . | indent 12 }}
-          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-            imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-volumes
-            args:
-            - /bin/bash
-            - -c
-            - python -u /workspace/manage.py measure_volumes
-            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          volumes:
+          - name: controller-config
+            configMap:
+              name: controller-config
diff --git a/charts/controller/templates/controller-cronjob-hourly.yaml b/charts/controller/templates/controller-cronjob-hourly.yaml
index afe9aa0f1..610cdd803 100644
--- a/charts/controller/templates/controller-cronjob-hourly.yaml
+++ b/charts/controller/templates/controller-cronjob-hourly.yaml
@@ -16,8 +16,9 @@ spec:
     spec:
       template:
         metadata:
-          labels:
-            app: drycc-controller-conjob
+          labels: {{- include "common.labels.standard" . | nindent 12 }}
+            app: drycc-controller
+            component: drycc-controller-conjob-hourly
         spec:
           restartPolicy: OnFailure
           serviceAccount: drycc-controller
@@ -29,24 +30,102 @@ spec:
               - netcat
               - -v
               - -u
-              - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-              - -a
-              - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
-            {{- include "controller.envs" . | indent 12 }}
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
+              - -u
+              - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
+            {{- include "controller-job.envs" . | indent 12 }}
           containers:
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-networks
+            name: drycc-controller-upload-network-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_networks
+            - python -u /workspace/manage.py upload_network_usage
+            {{- end }}
             {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
           - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
             imagePullPolicy: {{.Values.pull_policy}}
-            name: drycc-controller-measure-loadbalancer
+            name: drycc-controller-upload-gateway-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
             args:
             - /bin/bash
             - -c
-            - python -u /workspace/manage.py measure_loadbalancers
-            {{- include "controller.envs" . | indent 12 }}
\ No newline at end of file
+            - python -u /workspace/manage.py upload_gateway_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-upload-app-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
+            args:
+              - /bin/bash
+              - -c
+              - python -u /workspace/manage.py upload_app_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-upload-resources-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
+            args:
+            - /bin/bash
+            - -c
+            - python -u /workspace/manage.py upload_resource_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          - image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+            imagePullPolicy: {{.Values.pull_policy}}
+            name: drycc-controller-upload-volume-usage
+            {{- if .Values.diagnosticMode.enabled }}
+            command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 14 }}
+            args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 14 }}
+            {{- else }}
+            args:
+            - /bin/bash
+            - -c
+            - python -u /workspace/manage.py upload_volume_usage
+            {{- end }}
+            {{- include "controller.envs" . | indent 12 }}
+            {{- include "controller-job.envs" . | indent 12 }}
+            volumeMounts:
+            - name: controller-config
+              readOnly: false
+              mountPath: /etc/controller
+          volumes:
+          - name: controller-config
+            configMap:
+              name: controller-config
diff --git a/charts/controller/templates/controller-job-upgrade.yaml b/charts/controller/templates/controller-job-upgrade.yaml
new file mode 100644
index 000000000..133e534ad
--- /dev/null
+++ b/charts/controller/templates/controller-job-upgrade.yaml
@@ -0,0 +1,49 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: drycc-controller-job-upgrade
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
+    helm.sh/hook: post-install,post-upgrade,post-rollback
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+spec:
+  template:
+    spec:
+      serviceAccount: drycc-controller
+      initContainers:
+      - name: drycc-controller-job-upgrade-init
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        args:
+        - netcat
+        - -v
+        - -u
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
+        {{- include "controller.envs" . | indent 8 }}
+      containers:
+      - name: drycc-controller-job-upgrade
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        {{- if .Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 8 }}
+        args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 8 }}
+        {{- else }}
+        args:
+        - /usr/bin/env
+        - bash
+        - -ec
+        - |
+          python -u /workspace/manage.py migrate --noinput
+          python -u /workspace/manage.py loaddata /etc/controller/limit-specs.json
+          python -u /workspace/manage.py loaddata /etc/controller/limit-plans.json
+        {{- end }}
+        {{- include "controller.envs" . | indent 8 }}
+        volumeMounts:
+        - name: controller-config
+          readOnly: false
+          mountPath: /etc/controller
+      volumes:
+      - name: controller-config
+        configMap:
+          name: controller-config
+      restartPolicy: Never
diff --git a/charts/controller/templates/controller-secret-creds.yaml b/charts/controller/templates/controller-secret-creds.yaml
index 0b71b86e3..0e78d51c6 100644
--- a/charts/controller/templates/controller-secret-creds.yaml
+++ b/charts/controller/templates/controller-secret-creds.yaml
@@ -5,17 +5,17 @@ metadata:
   labels:
     heritage: drycc
 data:
+  {{- if (.Values.valkeyUrl) }}
+  valkey-url: {{ .Values.valkeyUrl | b64enc }}
+  {{- end }}
   {{- if (.Values.databaseUrl) }}
   database-url: {{ .Values.databaseUrl | b64enc }}
   {{- end }}
   {{- if (.Values.databaseReplicaUrl) }}
   database-replica-url: {{ .Values.databaseReplicaUrl | b64enc }}
   {{- end }}
-  {{- if (.Values.databaseMonitorUrl) }}
-  database-monitor-url: {{ .Values.databaseMonitorUrl | b64enc }}
-  {{- end }}
-  {{- if (.Values.rabbitmqUrl) }}
-  rabbitmq-url: {{ .Values.rabbitmqUrl | b64enc }}
+  {{- if (.Values.victoriametricsUrl) }}
+  victoriametrics-url: {{ .Values.victoriametricsUrl | b64enc }}
   {{- end }}
   {{- if (.Values.passportUrl) }}
   passport-url: {{ .Values.passportUrl | b64enc }}
@@ -26,5 +26,11 @@ data:
   {{- if (.Values.passportSecret) }}
   passport-secret: {{ .Values.passportSecret | b64enc }}
   {{- end }}
-  django-secret-key: {{ randAscii 64 | b64enc }}
-  deploy-hook-secret-key: {{ randAscii 64 | b64enc }}
\ No newline at end of file
+  {{- if (.Values.registryHost) }}
+  registry-host: {{ .Values.registryHost | b64enc }}
+  registry-username: {{ .Values.registryUsername | b64enc }}
+  registry-password: {{ .Values.registryPassword | b64enc }}
+  {{- end }}
+  service-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "service-key" "defaultValue" (randAlphaNum 64) "context" $)) }}
+  django-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "django-secret-key" "defaultValue" (randAlphaNum 64) "context" $)) }}
+  deploy-hook-secret-key: {{ (include "common.secrets.lookup" (dict "secret" "controller-creds" "key" "deploy-hook-secret-key" "defaultValue" (randAlphaNum 64) "context" $)) }}
diff --git a/charts/controller/templates/controller-webhook-deloyment.yaml b/charts/controller/templates/controller-webhook-deloyment.yaml
deleted file mode 100644
index 6375ae1d3..000000000
--- a/charts/controller/templates/controller-webhook-deloyment.yaml
+++ /dev/null
@@ -1,68 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: drycc-controller-webhook
-spec:
-  replicas: {{ .Values.webhookReplicas }}
-  selector:
-    matchLabels:
-      component: drycc-controller-webhook
-  template:
-    metadata:
-      labels: {{- include "common.labels.standard" . | nindent 8 }}
-        component: drycc-controller-webhook
-    spec:
-      affinity:
-        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.webhook.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.webhook.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
-        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.webhook.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.webhook.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
-        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.webhook.nodeAffinityPreset.type "key" .Values.webhook.nodeAffinityPreset.key "values" .Values.webhook.nodeAffinityPreset.values ) | nindent 10 }}
-      initContainers:
-      - name: drycc-controller-init
-        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
-        imagePullPolicy: {{.Values.imagePullPolicy}}
-        args:
-        - netcat
-        - -v
-        - -u
-        - $(DRYCC_DATABASE_URL),$(DRYCC_DATABASE_REPLICA_URL),$(DRYCC_RABBITMQ_URL)
-        - -a
-        - $(DRYCC_REDIS_ADDRS),$(DRYCC_CONTROLLER_API_SERVICE_HOST):$(DRYCC_CONTROLLER_API_SERVICE_PORT)
-        {{- include "controller.envs" . | indent 8 }}
-      containers:
-      - name: drycc-controller
-        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
-        imagePullPolicy: {{.Values.imagePullPolicy}}
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8443
-            scheme: HTTPS
-          initialDelaySeconds: 30
-          timeoutSeconds: 10
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8443
-            scheme: HTTPS
-          initialDelaySeconds: 30
-          timeoutSeconds: 10
-          periodSeconds: 5
-        ports:
-          - containerPort: 8443
-            name: https
-        volumeMounts:
-        - name: drycc-controller-webhook-cert
-          mountPath: /etc/controller/webhook/cert
-        {{- include "controller.limits" . | indent 8 }}
-        {{- include "controller.envs" . | indent 8 }}
-      volumes:
-      - name: drycc-controller-webhook-cert                                                                                                                                                                                               
-        secret:                                                                                                                                                                                                                           
-          secretName: drycc-controller-webhook-cert                                                                                                                                                                                       
-          items:  
-          - key: token                                                                                                                                                                                                                  
-            path: token                                                                                                                                                                                                                           
-          - key: tls.crt                                                                                                                                                                                                                  
-            path: tls.crt                                                                                                                                                                                                                 
-          - key: tls.key                                                                                                                                                                                                                  
-            path: tls.key
\ No newline at end of file
diff --git a/charts/controller/templates/controller-webhook-register.yaml b/charts/controller/templates/controller-webhook-register.yaml
deleted file mode 100644
index 3b0644414..000000000
--- a/charts/controller/templates/controller-webhook-register.yaml
+++ /dev/null
@@ -1,40 +0,0 @@
-{{- $token := randAlphaNum 128 | lower }}
-{{- $ca := genCA "controller-webhook-ca" 3650 }}
-{{- $altName1 := printf "drycc-controller-webhook.%s" .Release.Namespace }}
-{{- $altName2 := printf "drycc-controller-webhook.%s.svc" .Release.Namespace }}
-{{- $cert := genSignedCert "drycc-controller-webhook" nil (list $altName1 $altName2) 3650 $ca }}
-apiVersion: admissionregistration.k8s.io/v1
-kind: MutatingWebhookConfiguration
-metadata:
-  name: drycc-controller-webhook
-webhooks:
-- name: "{{ $altName2 }}"
-  sideEffects: None
-  admissionReviewVersions: ["v1"]
-  clientConfig:
-    caBundle: {{ b64enc $ca.Cert }}
-    service:
-      name: drycc-controller-webhook
-      namespace: "{{ .Release.Namespace }}"
-      path: "{{ printf "/v2/webhooks/scale/%s/" $token }}"
-      port: 8443
-  failurePolicy: Fail
-  rules:
-  - operations: ["UPDATE"]
-    apiGroups: ["apps"]
-    apiVersions: ["*"]
-    resources: ["deployments/scale"]
-  timeoutSeconds: 30
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: drycc-controller-webhook-cert
-  labels:
-    release: "{{ .Release.Name }}"
-    heritage: "{{ .Release.Service }}"
-type: Opaque
-data:
-  token: {{ b64enc $token }}
-  tls.crt: {{ b64enc $cert.Cert }}
-  tls.key: {{ b64enc $cert.Key }}
diff --git a/charts/controller/templates/controller-webhook-service.yaml b/charts/controller/templates/controller-webhook-service.yaml
deleted file mode 100644
index c1c04b9b6..000000000
--- a/charts/controller/templates/controller-webhook-service.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-kind: Service
-apiVersion: v1
-metadata:
-  name: drycc-controller-webhook
-spec:
-  selector:
-    component: drycc-controller-webhook
-  ports:
-  - name: https
-    port: 8443
-    targetPort: 8443
\ No newline at end of file
diff --git a/charts/controller/templates/metric/controller-metric-deployment.yaml b/charts/controller/templates/metric/controller-metric-deployment.yaml
new file mode 100644
index 000000000..7687889f9
--- /dev/null
+++ b/charts/controller/templates/metric/controller-metric-deployment.yaml
@@ -0,0 +1,87 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: drycc-controller-metric
+  labels:
+    heritage: drycc
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
+spec:
+  replicas: {{ .Values.metric.replicas }}
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: drycc-controller
+      component: drycc-controller-metric
+  template:
+    metadata:
+      labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app: drycc-controller
+        component: drycc-controller-metric
+    spec:
+      affinity:
+        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.metric.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.metric.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
+        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.metric.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.metric.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
+        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.metric.nodeAffinityPreset.type "key" .Values.metric.nodeAffinityPreset.key "values" .Values.metric.nodeAffinityPreset.values ) | nindent 10 }}
+      serviceAccount: drycc-controller
+      initContainers:
+      - name: drycc-controller-metric-init
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        args:
+        - netcat
+        - -v
+        - -u
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL)
+        {{- include "controller-metric.envs" . | indent 8 }}
+      containers:
+      - name: drycc-controller-metric
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        {{- if .Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
+        args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if not .Values.diagnosticMode.enabled }}
+        startupProbe:
+          tcpSocket:
+            port: 8000
+          initialDelaySeconds: 30
+          periodSeconds: 60
+          timeoutSeconds: 1
+          successThreshold: 1
+          failureThreshold: 10
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8000
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 8000
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+          periodSeconds: 5
+        {{- end }}
+        ports:
+          - containerPort: 8000
+            name: http
+        {{- with index .Values "metric" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- include "controller-metric.envs" . | indent 8 }}
+        volumeMounts:
+        - name: controller-config
+          readOnly: false
+          mountPath: /etc/controller
+      volumes:
+      - name: controller-config
+        configMap:
+          name: controller-config
\ No newline at end of file
diff --git a/charts/controller/templates/metric/controller-metric-networkpolicy.yaml b/charts/controller/templates/metric/controller-metric-networkpolicy.yaml
new file mode 100644
index 000000000..5ee13a7e3
--- /dev/null
+++ b/charts/controller/templates/metric/controller-metric-networkpolicy.yaml
@@ -0,0 +1,19 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: drycc-controller-metric
+spec:
+  podSelector:
+    matchLabels:
+      app: drycc-controller
+      component: drycc-controller-metric
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    - namespaceSelector:
+        matchLabels:
+          kubernetes.io/metadata.name: {{.Release.Namespace}}
+      podSelector:
+        matchLabels:
+          app: drycc-victoriametrics
diff --git a/charts/controller/templates/metric/controller-metric-service.yaml b/charts/controller/templates/metric/controller-metric-service.yaml
new file mode 100644
index 000000000..8151a0cb7
--- /dev/null
+++ b/charts/controller/templates/metric/controller-metric-service.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: drycc-controller-metric
+  annotations:
+    {{- with .Values.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    app: drycc-controller
+    heritage: drycc
+    component: drycc-controller-metric
+spec:
+  clusterIP: None
+  ports:
+    - name: http
+      port: 8000
+      targetPort: 8000
+  selector:
+    app: drycc-controller
+    component: drycc-controller-metric
diff --git a/charts/controller/templates/mutate/controller-mutate-deloyment.yaml b/charts/controller/templates/mutate/controller-mutate-deloyment.yaml
new file mode 100644
index 000000000..3004bd684
--- /dev/null
+++ b/charts/controller/templates/mutate/controller-mutate-deloyment.yaml
@@ -0,0 +1,70 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: drycc-controller-mutate
+spec:
+  replicas: {{ .Values.mutate.replicas }}
+  selector:
+    matchLabels:
+      app: drycc-controller
+      component: drycc-controller-mutate
+  template:
+    metadata:
+      labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app: drycc-controller
+        component: drycc-controller-mutate
+    spec:
+      affinity:
+        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.mutate.podAffinityPreset.type "component" "" "extraMatchLabels" .Values.mutate.podAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
+        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.mutate.podAntiAffinityPreset.type "component" "" "extraMatchLabels" .Values.mutate.podAntiAffinityPreset.extraMatchLabels "topologyKey" "" "context" $) | nindent 10 }}
+        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.mutate.nodeAffinityPreset.type "key" .Values.mutate.nodeAffinityPreset.key "values" .Values.mutate.nodeAffinityPreset.values ) | nindent 10 }}
+      initContainers:
+      - name: drycc-controller-mutate-init
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/python-dev:latest
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        args:
+        - netcat
+        - -v
+        - -u
+        - $(DRYCC_DATABASE_URL),$(DRYCC_VALKEY_URL),$(DRYCC_DATABASE_REPLICA_URL),http://drycc-controller-api
+        {{- include "controller-mutate.envs" . | indent 8 }}
+      containers:
+      - name: drycc-controller-mutate
+        image: {{.Values.imageRegistry}}/{{.Values.imageOrg}}/controller:{{.Values.imageTag}}
+        imagePullPolicy: {{.Values.imagePullPolicy}}
+        {{- if .Values.diagnosticMode.enabled }}
+        command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 10 }}
+        args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 10 }}
+        {{- end }}
+        {{- if not .Values.diagnosticMode.enabled }}
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 8443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+          periodSeconds: 5
+        ports:
+          - containerPort: 8443
+            name: https
+        {{- end }}
+        volumeMounts:
+        - name: controller-mutate-cert
+          mountPath: /etc/controller/cert
+        {{- with index .Values "mutate" "resources" }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- include "controller-mutate.envs" . | indent 8 }}
+      volumes:
+      - name: controller-mutate-cert                                                                                                                                                                                               
+        secret:                                                                                                                                                                                                                           
+          secretName: controller-mutate-cert
\ No newline at end of file
diff --git a/charts/controller/templates/mutate/controller-mutate-service.yaml b/charts/controller/templates/mutate/controller-mutate-service.yaml
new file mode 100644
index 000000000..120df9a6c
--- /dev/null
+++ b/charts/controller/templates/mutate/controller-mutate-service.yaml
@@ -0,0 +1,20 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: drycc-controller-mutate
+  annotations:
+    {{- with .Values.service.annotations }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  labels:
+    app: drycc-controller
+    heritage: drycc
+    component: drycc-controller-mutate
+spec:
+  selector:
+    app: drycc-controller
+    component: drycc-controller-mutate
+  ports:
+  - name: https
+    port: 443
+    targetPort: 8443
\ No newline at end of file
diff --git a/charts/controller/templates/mutate/controller-mutate-webhook.yaml b/charts/controller/templates/mutate/controller-mutate-webhook.yaml
new file mode 100644
index 000000000..bc7eb614b
--- /dev/null
+++ b/charts/controller/templates/mutate/controller-mutate-webhook.yaml
@@ -0,0 +1,57 @@
+{{- $key := randAlphaNum 128 | lower }}
+{{- $ca := genCA "controller-mutate-ca" 3650 }}
+{{- $altName1 := printf "drycc-controller-mutate.%s" .Release.Namespace }}
+{{- $altName2 := printf "drycc-controller-mutate.%s.svc" .Release.Namespace }}
+{{- $cert := genSignedCert "drycc-controller-mutate" nil (list $altName1 $altName2) 3650 $ca }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: {{ .Release.Namespace }}-controller-mutate
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
+  labels:
+    heritage: drycc
+webhooks:
+- name: mutate.drycc.cc
+  sideEffects: None
+  admissionReviewVersions: ["v1"]
+  clientConfig:
+    caBundle: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "ca.crt" "defaultValue" $ca.Cert "context" $)) }}
+    service:
+      name: drycc-controller-mutate
+      namespace: "{{ .Release.Namespace }}"
+      path: "{{ printf "/v2/mutate/%s/" ((include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "key" "defaultValue" $key "context" $)) | b64dec) }}"
+  failurePolicy: Fail
+  objectSelector:
+    matchLabels:
+      heritage: drycc
+  rules:
+  - operations: ["UPDATE"]
+    apiGroups: ["batch"]
+    apiVersions: ["*"]
+    resources: ["jobs/status"]
+  - operations: ["UPDATE"]
+    apiGroups: ["apps"]
+    apiVersions: ["*"]
+    resources: ["deployments/scale"]
+  - operations: ["UPDATE"]
+    apiGroups: ["servicecatalog.k8s.io"]
+    apiVersions: ["*"]
+    resources: ["serviceinstances/status", "servicebindings/status"]
+  timeoutSeconds: 30
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: controller-mutate-cert
+  annotations:
+    component.drycc.cc/version: {{ .Values.imageTag }}
+  labels:
+    heritage: drycc
+type: Opaque
+data:
+  key: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "key" "defaultValue" $key "context" $)) }}
+  ca.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "ca.key" "defaultValue" $ca.Key "context" $)) }}
+  ca.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "ca.crt" "defaultValue" $ca.Cert "context" $)) }}
+  tls.key: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "tls.key" "defaultValue" $cert.Key "context" $)) }}
+  tls.crt: {{ (include "common.secrets.lookup" (dict "secret" "controller-mutate-cert" "key" "tls.crt" "defaultValue" $cert.Cert "context" $)) }}
diff --git a/charts/controller/values.yaml b/charts/controller/values.yaml
index 090edfabb..2a93156ff 100644
--- a/charts/controller/values.yaml
+++ b/charts/controller/values.yaml
@@ -2,13 +2,24 @@ imageOrg: "drycc"
 imagePullPolicy: "Always"
 imageTag: "canary"
 imageRegistry: "registry.drycc.cc"
-appImagePullPolicy: "Always"
+## Enable diagnostic mode
+##
+diagnosticMode:
+  ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)
+  ##
+  enabled: false
+  ## @param diagnosticMode.command Command to override all containers
+  ##
+  command:
+    - sleep
+  ## @param diagnosticMode.args Args to override all containers
+  ##
+  args:
+    - infinity
 # A comma-separated list of URLs to send app release information to
 # See https://www.drycc.cc/managing-workflow/deploy-hooks/
 deployHookUrls: ""
 timezone: "UTC"
-# limitsCpu: "100m"
-# limitsMemory: "50Mi"
 # Possible values are:
 # enabled - allows for open registration
 # disabled - turns off open registration
@@ -16,9 +27,15 @@ timezone: "UTC"
 registrationMode: "admin_only"
 # Option to disable ssl verification to connect to k8s api server
 k8sApiVerifyTls: "true"
-
-# Set storageClassName. It is used for application.
-appStorageClass: "{{ .Values.storage.csi.storageClassName }}"
+# Set app image pull policy
+appImagePullPolicy: "Always"
+# Filer settings image and image pull policy
+filerImage: "registry.drycc.cc/drycc/filer:canary"
+filerImagePullPolicy: "Always"
+# Set app gateway, It is used for application gateway.
+appGatewayClass: ""
+# Set storageClassName, It is used for application mount.
+appStorageClass: "longhorn"
 # Set runtimeClassName. It is used for application.
 appRuntimeClass: ""
 # Set appDNSPolicy. It is used for application.
@@ -26,17 +43,13 @@ appRuntimeClass: ""
 appDNSPolicy: ""
 # set timeout seconds. It is used for pod exec
 appPodExecTimeout: 3600
-# Set api replicas
-apiReplicas: 1
-# Set celery replicas
-celeryReplicas: 1
-# Set webhook replicas
-webhookReplicas: 1
 # Set cronjob concurrencyPolicy
 # Allow (default): The cron job allows concurrently running jobs
 # Forbid: The cron job does not allow concurrent runs; if it is time for a new job run and the previous job run hasn't finished yet, the cron job skips the new job run
 # Replace: If it is time for a new job run and the previous job run hasn't finished yet, the cron job replaces the currently running job run with a new job run
 concurrencyPolicy: "Replace"
+## valkeyUrl is will no longer use the built-in valkey component
+valkeyUrl: ""
 # databaseUrl and databaseReplicaUrl are will no longer use the built-in database component
 databaseUrl: ""
 databaseReplicaUrl: ""
@@ -44,24 +57,46 @@ databaseReplicaUrl: ""
 passportUrl: ""
 passportKey: ""
 passportSecret: ""
+# victoriametricsUrl is will no longer use the built-in victoriametrics component
+# format: http://victoriametrics-server/select/0/prometheus
+victoriametricsUrl: ""
 # Workflow-manager Configuration Options
 workflowManagerUrl: ""
 workflowManagerAccessKey: ""
 workflowManagerSecretKey: ""
+# Prefix for the imagepull secret created when using private registry
+registrySecretPrefix: "private-registry"
+# limit specs, plans config
+config:
+  metrics: ""
+  limitSpecs: ""
+  limitPlans: ""
+  reservedNames: ""
+  secretTemplate: ""
+  volumeTemplate: ""
+  volumeClaimTemplate: ""
+  volumeUsageTemplate: ""
+  networkUsageTemplate: ""
 
-# Configuring this will no longer use the built-in rabbitmq component
-rabbitmqUrl: ""
 # Service
 service:
   # Provide any additional service annotations
   annotations: {}
+
 # Any custom controller environment variables
 # can be specified as key-value pairs under environment
 # this is usually a non required setting.
-environment:
-  RESERVED_NAMES: "drycc, drycc-builder, drycc-monitor-grafana, drycc-passport, drycc-helmbroker, drycc-manager"
+environment: {}
 
 api:
+  replicas: 1
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
   nodeAffinityPreset:
     key: "drycc.cc/node"
     type: "soft"
@@ -77,6 +112,14 @@ api:
       app: "drycc-controller-api"
 
 celery:
+  replicas: 1
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
   nodeAffinityPreset:
     key: "drycc.cc/node"
     type: "soft"
@@ -91,7 +134,15 @@ celery:
     extraMatchLabels:
       app: "drycc-controller-celery"
 
-webhook:
+metric:
+  replicas: 1
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
   nodeAffinityPreset:
     key: "drycc.cc/node"
     type: "soft"
@@ -104,60 +155,55 @@ webhook:
   podAntiAffinityPreset:
     type: "soft"
     extraMatchLabels:
-      component: "drycc-controller-webhook"
+      component: "drycc-controller-metric"
 
-redis:
+mutate:
   replicas: 1
+  resources: {}
+    # limits:
+    #   cpu: 200m
+    #   memory: 50Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 30Mi
+  nodeAffinityPreset:
+    key: "drycc.cc/node"
+    type: "soft"
+    values:
+    - "true"
+  podAffinityPreset:
+    type: ""
+    extraMatchLabels:
+      security: "drycc-security"
+  podAntiAffinityPreset:
+    type: "soft"
+    extraMatchLabels:
+      component: "drycc-controller-mutate"
+
+valkey:
+  enabled: true
+
+database:
+  enabled: true
 
-storage:
-  csi:
-    storageClassName: drycc-storage
+quickwit:
+  logIndexPrefix: logs-
+
+registry:
+  enabled: true
+
+passport:
+  enabled: true
+
+victoriametrics:
+  enabled: true
 
 global:
   # Admin email, used for each component to send email to administrator
   email: "drycc@drycc.cc"
-  # Set the storage backend
-  #
-  # Valid values are:
-  # - s3: Store persistent data in AWS S3 (configure in S3 section)
-  # - azure: Store persistent data in Azure's object storage
-  # - gcs: Store persistent data in Google Cloud Storage
-  # - minio: Store persistent data on in-cluster Minio server
-  storage: minio
-  # Set the location of Workflow's PostgreSQL database
-  #
-  # Valid values are:
-  # - on-cluster: Run PostgreSQL within the Kubernetes cluster (credentials are generated
-  #   automatically; backups are sent to object storage
-  #   configured above)
-  # - off-cluster: Run PostgreSQL outside the Kubernetes cluster (configure in database section)
-  databaseLocation: "on-cluster"
-
-  # Set the location of Workflow's Registry
-  #
-  # Valid values are:
-  # - on-cluster: Run registry within the Kubernetes cluster
-  # - off-cluster: Use registry outside the Kubernetes cluster (example: drycc registry,self-hosted)
-  registryLocation: "on-cluster"
-  # The host port to which registry proxy binds to
-  registryProxyPort: 5555
-  # Prefix for the imagepull secret created when using private registry
-  registrySecretPrefix: "private-registry"
-  # Role-Based Access Control for Kubernetes >= 1.5
-  rbac: false
-  # GatewayClass is cluster-scoped resource defined by the infrastructure provider.
-  gatewayClass: ""
-  # Please check `kubernetes.io/ingress.class`
-  ingressClass: ""
-  # A domain name consists of one or more parts.
-  # Periods (.) are used to separate these parts.
-  # Each part must be 1 to 63 characters in length and can contain lowercase letters, digits, and hyphens (-).
-  # It must start and end with a lowercase letter or digit.
-  clusterDomain: "cluster.local"
   # The public resolvable hostname to build your cluster with.
   #
   # This will be the hostname that is used to build endpoints such as "drycc.$HOSTNAME"
   platformDomain: ""
   # Whether cert_manager is enabled to automatically generate controller certificates
   certManagerEnabled: true
-  passportLocation: "on-cluster"
diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile
index 9c702bd34..ae5a3b607 100644
--- a/rootfs/Dockerfile
+++ b/rootfs/Dockerfile
@@ -1,10 +1,10 @@
 ARG CODENAME
 FROM registry.drycc.cc/drycc/base:${CODENAME}
 
-ENV DRYCC_UID=1001 \
+ARG DRYCC_UID=1001 \
   DRYCC_GID=1001 \
   DRYCC_HOME_DIR=/workspace \
-  PYTHON_VERSION="3.11"
+  PYTHON_VERSION="3.14"
 
 RUN groupadd drycc --gid ${DRYCC_GID} \
   && useradd drycc -u ${DRYCC_UID} -g ${DRYCC_GID} -s /bin/bash -m -d ${DRYCC_HOME_DIR}
@@ -12,7 +12,7 @@ RUN groupadd drycc --gid ${DRYCC_GID} \
 COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt
 
 RUN buildDeps='gcc libffi-dev libpq-dev rustc cargo'; \
-    install-packages ${buildDeps} \
+    install-packages inotify-tools ${buildDeps} \
   && install-stack python $PYTHON_VERSION && . init-stack \
   && python3 -m venv ${DRYCC_HOME_DIR}/.venv \
   && source ${DRYCC_HOME_DIR}/.venv/bin/activate \
diff --git a/rootfs/Dockerfile.test b/rootfs/Dockerfile.test
index 042ff5f0c..74f4c8c91 100644
--- a/rootfs/Dockerfile.test
+++ b/rootfs/Dockerfile.test
@@ -1,23 +1,23 @@
 ARG CODENAME
 FROM registry.drycc.cc/drycc/base:${CODENAME}
 
-ARG DRYCC_HOME_DIR=/workspace
+ARG DRYCC_HOME_DIR=/workspace \
+  PYTHON_VERSION="3.14" \
+  VALKEY_VERSION="9.0.0" \
+  POSTGRES_VERSION="18.1" \
+  GOSU_VERSION="1.19"
 
 COPY requirements.txt ${DRYCC_HOME_DIR}/requirements.txt
 COPY dev_requirements.txt ${DRYCC_HOME_DIR}/dev_requirements.txt
 
-ENV PGDATA="/opt/drycc/postgresql/data" \
-  PYTHON_VERSION="3.11" \
-  REDIS_VERSION="7.0.11" \
-  RABBITMQ_VERSION="3.12.0" \
-  POSTGRES_VERSION="15.3" \
-  GOSU_VERSION="1.16"
+ENV PGDATA="/opt/drycc/postgresql/data"
 
 RUN buildDeps='gcc rustc cargo  libffi-dev musl-dev openssl'; \
-  install-packages mercurial ca-certificates git $buildDeps \
+  install-packages mercurial ca-certificates git inotify-tools $buildDeps \
+  && curl -SsL https://cli.codecov.io/latest/$([ $(dpkg --print-architecture) == "arm64" ] && echo linux-arm64 || echo linux)/codecov -o /usr/local/bin/codecov \
+  && chmod +x /usr/local/bin/codecov \
   && install-stack python $PYTHON_VERSION \
-  && install-stack redis $REDIS_VERSION \
-  && install-stack rabbitmq $RABBITMQ_VERSION \
+  && install-stack valkey $VALKEY_VERSION \
   && install-stack postgresql $POSTGRES_VERSION \
   && install-stack gosu $GOSU_VERSION && . init-stack \
   && python3 -m venv ${DRYCC_HOME_DIR}/.venv \
diff --git a/rootfs/api/admin.py b/rootfs/api/admin.py
index c3710a944..199161af6 100644
--- a/rootfs/api/admin.py
+++ b/rootfs/api/admin.py
@@ -6,9 +6,7 @@
 
 
 from django.contrib import admin
-from guardian.admin import GuardedModelAdmin
 
-from .models import App
 from .models import Build
 from .models import Config
 from .models import Domain
@@ -16,18 +14,6 @@
 from .models import Release
 
 
-class AppAdmin(GuardedModelAdmin):
-    """Set presentation options for :class:`~api.models.App` models
-    in the Django admin.
-    """
-    date_hierarchy = 'created'
-    list_display = ('id', 'owner')
-    list_filter = ('owner',)
-
-
-admin.site.register(App, AppAdmin)
-
-
 class BuildAdmin(admin.ModelAdmin):
     """Set presentation options for :class:`~api.models.Build` models
     in the Django admin.
diff --git a/rootfs/api/admissions.py b/rootfs/api/admissions.py
new file mode 100644
index 000000000..859f9b847
--- /dev/null
+++ b/rootfs/api/admissions.py
@@ -0,0 +1,141 @@
+from django.db.models import F, Func, Value, JSONField
+from rest_framework.request import Request
+
+from api import models
+
+
+class BaseHandler(object):
+
+    def detect(self, request: Request) -> bool:
+        raise NotImplementedError()
+
+    def handle(self, request: Request) -> bool:
+        raise NotImplementedError()
+
+
+class JobsStatusHandler(BaseHandler):
+
+    def detect(self, request: Request) -> bool:
+        group = request.get("resource", {}).get("group", None)
+        resource = "/".join([
+            request.get("resource", {}).get("resource", None),
+            request.get("subResource", ""),
+        ])
+        if (group, resource) == ("batch", "jobs/status"):
+            return True
+        return False
+
+    def handle(self, request: Request) -> bool:
+        app_id = request["object"]["metadata"]["namespace"]
+        app = models.app.App.objects.filter(id=app_id).first()
+        ptype = request["object"]["metadata"].get("labels", {}).get("type", "")
+        if app and ptype:
+            lock = app.lock()
+            try:
+                lock.acquire()
+                status = request["object"]["status"]
+                replicas = request["object"]["spec"].get("replicas", 0)
+                if "active" in status:
+                    replicas += 1
+                elif "succeeded" in status or "failed" in status:
+                    replicas -= 1
+                replicas = 0 if replicas < 0 else replicas
+                if app.structure.get(ptype, 0) != replicas:
+                    models.app.App.objects.filter(id=app.id).update(
+                        structure=Func(
+                            F("structure"),
+                            Value([ptype]),
+                            Value(replicas, JSONField()),
+                            function="jsonb_set",
+                        )
+                    )
+            finally:
+                lock.release()
+        return True
+
+
+class DeploymentsScaleHandler(BaseHandler):
+
+    def detect(self, request: Request) -> bool:
+        group = request.get("resource", {}).get("group", None)
+        resource = "/".join([
+            request.get("resource", {}).get("resource", None),
+            request.get("subResource", ""),
+        ])
+        if (group, resource) == ("apps", "deployments/scale"):
+            return True
+        return False
+
+    def handle(self, request: Request) -> bool:
+        app_id = request["object"]["metadata"]["namespace"]
+        app = models.app.App.objects.filter(id=app_id).first()
+        ptype = None
+        for item in request["object"]["status"]["selector"].split(","):
+            key, value = item.split("=")
+            if key == "type":
+                ptype = value
+        if app and ptype:
+            lock = app.lock()
+            try:
+                lock.acquire()
+                replicas = request["object"]["spec"].get("replicas", 0)
+                if app.structure.get(ptype, 0) != replicas:
+                    models.app.App.objects.filter(id=app.id).update(
+                        structure=Func(
+                            F("structure"),
+                            Value([ptype]),
+                            Value(replicas, JSONField()),
+                            function="jsonb_set",
+                        )
+                    )
+            finally:
+                lock.release()
+        return True
+
+
+class ServiceInstancesStatusHandler(BaseHandler):
+
+    def detect(self, request: Request) -> bool:
+        group = request.get("resource", {}).get("group", None)
+        resource = "/".join([
+            request.get("resource", {}).get("resource", None),
+            request.get("subResource", ""),
+        ])
+        if (group, resource) == ("servicecatalog.k8s.io", "serviceinstances/status"):
+            return True
+        return False
+
+    def handle(self, request: Request) -> bool:
+        app_id = request["object"]["metadata"]["namespace"]
+        name = request["object"]["metadata"]["name"]
+        status = request["object"]["status"]["lastConditionState"]
+        resource = models.resource.Resource.objects.filter(
+            app__id=app_id, name=name).first()
+        if resource and resource.status != status:
+            resource.status = status
+            resource.save(update_fields=["status"])
+        return True
+
+
+class ServicebindingsStatusHandler(BaseHandler):
+
+    def detect(self, request: Request) -> bool:
+        group = request.get("resource", {}).get("group", None)
+        resource = "/".join([
+            request.get("resource", {}).get("resource", None),
+            request.get("subResource", ""),
+        ])
+        if (group, resource) == ("servicecatalog.k8s.io", "servicebindings/status"):
+            return True
+        return False
+
+    def handle(self, request: Request) -> bool:
+        app_id = request["object"]["metadata"]["namespace"]
+        name = request["object"]["metadata"]["name"]
+        binding = request["object"]["status"]["lastConditionState"]
+        resource = models.resource.Resource.objects.filter(
+            app__id=app_id, name=name).first()
+        if resource and resource.binding != binding:
+            resource.binding = binding
+            resource.save(update_fields=["binding"])
+        return True
diff --git a/rootfs/api/apps_extra/social_core/actions.py b/rootfs/api/apps_extra/social_core/actions.py
index 379ce539b..c86bac74b 100644
--- a/rootfs/api/apps_extra/social_core/actions.py
+++ b/rootfs/api/apps_extra/social_core/actions.py
@@ -1,6 +1,5 @@
 import json
 from urllib.parse import quote
-
 from social_core.utils import (
     partial_pipeline_data,
     sanitize_redirect,
@@ -40,8 +39,9 @@ def form2json(form_data):
         query = urlparse("?" + form_data).query
         params = parse_qs(query)
         return {key: params[key][0] for key in params}
-    from django.core.cache import cache
-    cache.set("oidc_key_" + data.get("key", ""), form2json(url).get("state"), 60 * 10)
+    from api.apps_extra.social_core.backends import OauthCacheManager
+    oauth_cache_manager = OauthCacheManager()
+    oauth_cache_manager.set_state(data.get("key", ""), form2json(url).get("state"))
     return response
 
 
@@ -129,11 +129,9 @@ def do_complete(backend, login, user=None, redirect_name="next", *args, **kwargs
     if social_auth and social_auth.extra_data:
         extra_data = json.loads(social_auth.extra_data) if \
             isinstance(social_auth.extra_data, str) else social_auth.extra_data
-    from django.core.cache import cache
-    cache.set("oidc_state_" + data.get("state"),
-              {"token": extra_data.get("id_token", "fail"),
-               "username": user.username},
-              60 * 10)
+    from api.apps_extra.social_core.backends import OauthCacheManager
+    oauth_cache_manager = OauthCacheManager()
+    oauth_cache_manager.set_token(data.get("state"), extra_data)
     return response
 
 
diff --git a/rootfs/api/apps_extra/social_core/backends.py b/rootfs/api/apps_extra/social_core/backends.py
new file mode 100644
index 000000000..aa73b9281
--- /dev/null
+++ b/rootfs/api/apps_extra/social_core/backends.py
@@ -0,0 +1,86 @@
+import logging
+from django.conf import settings
+from django.core.cache import cache
+from django.contrib.auth import get_user_model
+from django.utils.translation import gettext_lazy
+
+from social_core.backends.open_id_connect import OpenIdConnectAuth
+from rest_framework import exceptions
+
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+
+
+class DryccOIDC(OpenIdConnectAuth):
+    """Drycc Openid Connect authentication backend"""
+    name = 'drycc'
+    EXTRA_DATA = [
+        ('id', 'id'),
+        ('access_token', 'access_token'),
+        ('refresh_token', 'refresh_token'),
+        ('expires_in', 'expires_in'),
+        ('token_type', 'token_type'),
+        ('id_token', 'id_token'),
+        ('scope', 'scope'),
+    ]
+
+    def get_user_data(self, access_token):
+        """Loads user data from service"""
+        response = self.get_json(
+            self.userinfo_url(),
+            headers={
+                'authorization': 'Bearer ' + access_token
+            },
+        )
+        return {
+            'id': response.get('id'),
+            'username': response.get('username'),
+            'email': response.get('email') or '',
+            'first_name': response.get('first_name'),
+            'last_name': response.get('last_name'),
+            'is_superuser': response.get('is_superuser'),
+            'is_staff': response.get('is_staff'),
+            'is_active': response.get('is_active'),
+        }
+
+    def refresh_token(self, refresh_token):
+        # Get token URL from OIDC discovery if not already cached
+        return self.get_json(
+            self.access_token_url(),
+            method='POST',
+            data={
+                'grant_type': 'refresh_token',
+                'client_id': self.get_key_and_secret()[0],
+                'refresh_token': refresh_token,
+            },
+        )
+
+
+class OauthCacheManager(object):
+
+    def __init__(self):
+        self.drycc_oauth = DryccOIDC()
+
+    def get_user(self, access_token):
+        def _get_user(access_token):
+            from api import serializers
+            try:
+                user_info = self.drycc_oauth.get_user_data(access_token)
+                user, _ = serializers.UserSerializer.update_or_create(user_info)
+                return user
+            except Exception as e:
+                logger.info(e)
+                raise exceptions.AuthenticationFailed(gettext_lazy('Verify token fail.'))
+        return cache.get_or_set(
+            access_token, lambda: _get_user(access_token), settings.DRYCC_CACHE_USER_TIME)
+
+    def set_state(self, key, state):
+        cache.set("oidc_key_" + key, state, settings.DRYCC_CACHE_USER_TIME)
+
+    def set_token(self, state, data):
+        cache.set("oidc_state_" + state, data, settings.DRYCC_CACHE_USER_TIME)
+
+    def get_token(self, key):
+        state = cache.get("oidc_key_" + key, "")
+        return cache.get("oidc_state_" + state, {})
diff --git a/rootfs/api/apps_extra/social_core/pipelines.py b/rootfs/api/apps_extra/social_core/pipelines.py
new file mode 100644
index 000000000..5eefe0624
--- /dev/null
+++ b/rootfs/api/apps_extra/social_core/pipelines.py
@@ -0,0 +1,6 @@
+from api.serializers import UserSerializer
+
+
+def update_or_create(backend, user, response, *args, **kwargs):
+    user, created = UserSerializer.update_or_create(response)
+    return {'is_new': created, 'user': user}
diff --git a/rootfs/api/asgi.py b/rootfs/api/asgi.py
index 64eb77edb..4f0063cf8 100644
--- a/rootfs/api/asgi.py
+++ b/rootfs/api/asgi.py
@@ -9,18 +9,24 @@
 
 import os
 
+from django.urls import re_path
 from django.core.asgi import get_asgi_application
 from channels.routing import ProtocolTypeRouter, URLRouter
 from channels.security.websocket import AllowedHostsOriginValidator
 
 
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.settings.production')
-http = get_asgi_application()
-
-from .routing import websocket_urlpatterns  # noqa
-from .middleware import ChannelOAuthMiddleware  # noqa
-websocket = AllowedHostsOriginValidator(ChannelOAuthMiddleware(URLRouter(
-    websocket_urlpatterns
-)))
-
-application = ProtocolTypeRouter({"http": http, "websocket": websocket})
+django_asgi_app = get_asgi_application()
+
+from .routing import http_urlpatterns, websocket_urlpatterns  # noqa
+from .middleware import ChannelOAuthMiddleware, ChannelAPIVersionMiddleware  # noqa
+
+application = ProtocolTypeRouter({
+    "http": ChannelAPIVersionMiddleware(URLRouter([
+        *http_urlpatterns,
+        re_path(r'', django_asgi_app),
+    ])),
+    "websocket": AllowedHostsOriginValidator(ChannelOAuthMiddleware(URLRouter(
+        websocket_urlpatterns
+    )))
+})
diff --git a/rootfs/api/authentication.py b/rootfs/api/authentication.py
index e399947f5..63559903e 100644
--- a/rootfs/api/authentication.py
+++ b/rootfs/api/authentication.py
@@ -1,13 +1,10 @@
 import logging
-from django.conf import settings
 from django.contrib.auth.models import AnonymousUser
 from django.core.cache import cache
 from django.utils.translation import gettext_lazy
 from rest_framework import authentication
-from rest_framework.authentication import TokenAuthentication, \
-    get_authorization_header
+from rest_framework.authentication import get_authorization_header
 from rest_framework import exceptions
-from api.oauth import OAuthManager
 
 logger = logging.getLogger(__name__)
 
@@ -21,15 +18,16 @@ def authenticate(self, request):
         return AnonymousUser(), None
 
 
-class DryccAuthentication(TokenAuthentication):
+class DryccAuthentication(authentication.BaseAuthentication):
 
-    def authenticate(self, request):
-        if 'Drycc' in request.META.get('HTTP_USER_AGENT', ''):
-            auth = get_authorization_header(request).split()
-
-            if not auth or auth[0].lower() != self.keyword.lower().encode():
-                return None
+    keywords = ('token', 'bearer')
+    ignore_authentication_failed = False
 
+    def parse_header(self, request):
+        try:
+            auth = get_authorization_header(request).split()
+            if not auth or auth[0].decode().lower() not in self.keywords:
+                return None, None
             if len(auth) == 1:
                 msg = gettext_lazy('Invalid token header. No credentials provided.')
                 raise exceptions.AuthenticationFailed(msg)
@@ -37,26 +35,55 @@ def authenticate(self, request):
                 msg = gettext_lazy(
                     'Invalid token header. Token string should not contain spaces.')
                 raise exceptions.AuthenticationFailed(msg)
+            return auth[0].decode().lower(), auth[1].decode()
+        except UnicodeError:
+            msg = gettext_lazy(
+                'Invalid token header. Token string should not contain invalid characters.')
+            raise exceptions.AuthenticationFailed(msg)
 
+    def authenticate(self, request):
+        token_type, token = self.parse_header(request)
+        if token_type is None or token is None:
+            return None
+        try:
+            if token_type == 'bearer':  # drycc oauth access token
+                from api.apps_extra.social_core.backends import OauthCacheManager
+                return OauthCacheManager().get_user(token), token
+            # drycc token
+            user = cache.get(token, None)
+            if not user:
+                return self.authenticate_credentials(token)
+            return user, token
+        except exceptions.AuthenticationFailed as e:
+            if not self.ignore_authentication_failed:
+                raise e
+        return None
+
+    def authenticate_credentials(self, key):
+        from api.models.base import Token
+        try:
+            token = Token.objects.select_related('owner').get(key=key)
+        except Token.DoesNotExist:
+            raise exceptions.AuthenticationFailed(gettext_lazy('Invalid token.'))
+        if not token.owner.is_active:
+            raise exceptions.AuthenticationFailed(gettext_lazy('User inactive or deleted.'))
+        if token.expires():
             try:
-                token = auth[1].decode()
-            except UnicodeError:
-                msg = gettext_lazy(
-                    'Invalid token header. Token string should not contain invalid characters.')
-                raise exceptions.AuthenticationFailed(msg)
-            return cache.get_or_set(
-                token, lambda: self._get_user(token), settings.OAUTH_CACHE_USER_TIME), None
-        return super(DryccAuthentication, self).authenticate(request)
+                from api.apps_extra.social_core.backends import OauthCacheManager
+                user = OauthCacheManager().get_user(token.oauth['access_token'])
+                cache.set(key, user, timeout=token.oauth['expires_in'])
+                token.refresh_token()
+                return user, token.key
+            except exceptions.AuthenticationFailed:
+                return None, None
+        return (token.owner, token.key)
 
-    @staticmethod
-    def _get_user(key):
-        from api import serializers
+    def authenticate_header(self, request):
+        keyword = self.keywords[0]
         try:
-            user_info = OAuthManager().get_user_by_token(key)
-            if not user_info.get('email'):
-                user_info['email'] = OAuthManager().get_email_by_token(key)
-            user, _ = serializers.UserSerializer.update_or_create(user_info)
-            return user
-        except Exception as e:
-            logger.info(e)
-            raise exceptions.AuthenticationFailed(gettext_lazy('Verify token fail.'))
+            auth = self.parse_header(request)
+            if auth[0]:
+                keyword = auth[0]
+        except exceptions.AuthenticationFailed:
+            pass
+        return keyword
diff --git a/rootfs/api/backend.py b/rootfs/api/backend.py
deleted file mode 100644
index 41e8c3a18..000000000
--- a/rootfs/api/backend.py
+++ /dev/null
@@ -1,103 +0,0 @@
-from django.conf import settings
-from django.contrib.auth import get_user_model
-from django.core.exceptions import ObjectDoesNotExist
-
-from social_core.utils import cache
-from social_core.backends.oauth import BaseOAuth2
-from social_core.backends.open_id_connect import OpenIdConnectAuth
-
-from api import serializers
-from api.oauth import OAuthManager
-
-User = get_user_model()
-
-
-class DryccOauthBackend(object):
-
-    # The Django auth backend API
-    def authenticate(self, request, username=None, password=None, **kwargs):
-        if username is None:
-            return None
-
-        with OAuthManager() as client:
-            client.fetch_token(username, password)
-            user_info = client.get_user()
-            user_info['username'] = username
-            user_info['password'] = password
-            if not user_info.get('email'):
-                user_info['email'] = client.get_email()
-            user, _ = serializers.UserSerializer.update_or_create(user_info)
-        return user
-
-    def get_user(self, user_id):
-        user = None
-        try:
-            user = User.objects.get(pk=user_id)
-        except ObjectDoesNotExist:
-            pass
-        return user
-
-
-class DryccOAuth(BaseOAuth2):
-    """Drycc OAuth authentication backend"""
-    name = 'drycc'
-    AUTHORIZATION_URL = settings.SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL
-    ACCESS_TOKEN_URL = settings.SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL
-    ACCESS_TOKEN_METHOD = 'POST'
-    SCOPE_SEPARATOR = ','
-    EXTRA_DATA = [
-        ('id', 'id'),
-        ('access_token', 'access_token'),
-        ('refresh_token', 'refresh_token'),
-        ('expires_in', 'expires_in'),
-        ('token_type', 'token_type'),
-        ('id_token', 'id_token'),
-        ('scope', 'scope'),
-    ]
-
-    def get_user_details(self, response):
-        """Return user details from GitHub account"""
-        return {
-            'username': response.get('username'),
-            'email': response.get('email') or '',
-            'first_name': response.get('first_name'),
-            'last_name': response.get('last_name'),
-            'is_superuser': response.get('is_superuser'),
-            'is_staff': response.get('is_staff'),
-            'is_active': response.get('is_active'),
-        }
-
-    def user_data(self, access_token, *args, **kwargs):
-        """Loads user data from service"""
-        url = settings.SOCIAL_AUTH_DRYCC_USERINFO_URL
-        return self.get_json(url, headers={
-            'authorization': 'Bearer ' + access_token})
-
-    def get_user_id(self, details, response):
-        """Use user account id as unique id"""
-        return response.get('id')
-
-
-class DryccOIDC(OpenIdConnectAuth):
-    """Drycc Openid Connect authentication backend"""
-    name = 'drycc'
-    AUTHORIZATION_URL = settings.SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL
-    ACCESS_TOKEN_URL = settings.SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL
-    USERINFO_URL = settings.SOCIAL_AUTH_DRYCC_USERINFO_URL
-    JWKS_URI = settings.SOCIAL_AUTH_DRYCC_JWKS_URI
-    OIDC_ENDPOINT = settings.SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT
-    DEFAULT_SCOPE = ['openid']
-    EXTRA_DATA = [
-        ('id', 'id'),
-        ('access_token', 'access_token'),
-        ('refresh_token', 'refresh_token'),
-        ('expires_in', 'expires_in'),
-        ('token_type', 'token_type'),
-        ('id_token', 'id_token'),
-        ('scope', 'scope'),
-    ]
-
-    @cache(ttl=86400)
-    def oidc_config(self):
-        return self.get_json(self.OIDC_ENDPOINT +
-                             '/.well-known/openid-configuration/')
diff --git a/rootfs/api/consumers.py b/rootfs/api/consumers.py
index b32a2d050..47d680a83 100644
--- a/rootfs/api/consumers.py
+++ b/rootfs/api/consumers.py
@@ -1,71 +1,72 @@
 import json
 import time
+import six
+import ssl
 import aiohttp
 import asyncio
-import collections
+from collections import namedtuple
+from urllib.parse import urljoin
 from django.conf import settings
+from django.core.cache import cache
 
 from asgiref.sync import sync_to_async
 
-from kubernetes.client import Configuration
+from urllib3.response import HTTPResponse
+from kubernetes.client import Configuration, exceptions
 from kubernetes.client.api import core_v1_api
 from kubernetes.stream import stream
+from kubernetes.stream.ws_client import STDOUT_CHANNEL, STDERR_CHANNEL, ERROR_CHANNEL
 
-from channels.db import database_sync_to_async
 from channels.exceptions import DenyConnection
+from channels.generic.http import AsyncHttpConsumer
 from channels.generic.websocket import AsyncWebsocketConsumer
 
 from .models.app import App
-from .permissions import has_app_permission
+from .models.volume import Volume
+from .permissions import IsAppUser, get_app_status
 
 
-Request = collections.namedtuple("Request", ["user", "method"])
+class AppPermChecker(object):
+    timeout = 60 * 60
+    check_permission = IsAppUser().has_object_permission
 
-
-class BaseAppConsumer(AsyncWebsocketConsumer):
+    def __init__(self, scope):
+        self.scope = scope
 
     async def has_perm(self):
         if self.scope["user"] is None:
             return False, "user not login"
-        request = Request(self.scope["user"], "POST")
-        app = await database_sync_to_async(App.objects.get)(id=self.id)
-        return await database_sync_to_async(has_app_permission)(request, app)
+        app_id = self.scope["url_route"]["kwargs"]["id"]
+        key = f"permission:user:{self.scope["user"].id}:app:{app_id}"
+        permission = await cache.aget(key)
+        if permission is None:
+            try:
+                app = await App.objects.aget(id=app_id)
+                request = namedtuple("Request", ["user", "method"])(self.scope["user"], "GET")
+                if await sync_to_async(self.check_permission)(request, None, app):
+                    permission = await sync_to_async(get_app_status)(app)
+                else:
+                    permission = (False, "permission denied")
+                if permission[0]:
+                    await cache.aset(key, permission, timeout=self.timeout)
+            except App.DoesNotExist:
+                permission = (False, "app not exists")
+        return permission
+
+
+class BaseAppConsumer(AsyncWebsocketConsumer):
 
     async def connect(self):
-        self.id = self.scope["url_route"]["kwargs"]["id"]
-        is_ok, message = await self.has_perm()
+        app_perm_checker = AppPermChecker(self.scope)
+        is_ok, message = await app_perm_checker.has_perm()
         if is_ok:
             await self.accept()
+            self.id = self.scope["url_route"]["kwargs"]["id"]
         else:
             raise DenyConnection(message)
 
 
-class AppLogsConsumer(BaseAppConsumer):
-
-    async def receive(self, text_data=None, bytes_data=None):
-        if text_data is not None:
-            kwargs = json.loads(text_data)
-            lines = kwargs.get("lines", 100)
-            follow = kwargs.get("follow", False)
-            timeout = kwargs.get("timeout", 300)
-            url = "http://{}:{}/logs/{}?log_lines={}&follow={}&timeout={}".format(
-                settings.LOGGER_HOST,
-                settings.LOGGER_PORT,
-                self.id,
-                lines,
-                follow,
-                timeout,
-            )
-            async with aiohttp.ClientSession() as session:
-                async with session.get(url) as response:
-                    async for data in response.content.iter_any():
-                        await self.send(text_data=data)
-            await self.close(code=1000)
-        else:
-            raise ValueError("text_data cannot be empty!")
-
-
-class AppPodExecConsumer(BaseAppConsumer):
+class BaseK8sConsumer(BaseAppConsumer):
 
     @property
     def kubernetes(self):
@@ -79,53 +80,220 @@ def kubernetes(self):
         Configuration.set_default(config)
         return core_v1_api.CoreV1Api()
 
+
+class AppPodLogsConsumer(BaseK8sConsumer):
+
     async def connect(self):
-        self.stream = None
+        await super().connect()
+        self.running = False
+        self.response = None
         self.conneted = True
+        self.buffer = b''
+        self.delimiter = b"\r\n"
+        self.pod_name = self.scope["url_route"]["kwargs"]["name"]
+
+    def reader(self, sock):
+        self.buffer += sock.read()
+        try:
+            while self.buffer and self.buffer.endswith(self.delimiter):
+                index = self.buffer.index(self.delimiter)
+                length = int(self.buffer[:index], base=16)
+                if length == 0:
+                    asyncio.create_task(self.close(code=1000))
+                    break
+                start_pos = index + len(self.delimiter)
+                end_pos = start_pos + length + len(self.delimiter)
+                asyncio.create_task(
+                    self.send(bytes_data=self.buffer[start_pos:end_pos].strip(self.delimiter)))
+                self.buffer = self.buffer[end_pos:]
+        except BaseException:
+            asyncio.create_task(self.close(code=1000))
+
+    async def receive(self, text_data=None, bytes_data=None):
+        if self.running:
+            return
+        self.running = True
+        data = json.loads(text_data)
+        args = (self.pod_name, self.id)
+        lines = data.get("lines", 300)
+        follow = data.get("follow", False)
+        previous = data.get("previous", False)
+        kwargs = {
+            "follow": follow,
+            "container": data.get("container", ""),
+            "_preload_content": not follow,
+            "previous": previous
+        }
+        if lines > 0:
+            kwargs["tail_lines"] = lines
+        self.response = await sync_to_async(self.kubernetes.read_namespaced_pod_log)(
+            *args, **kwargs)
+        if follow:
+            loop = asyncio.get_event_loop()
+            loop.add_reader(
+                self.response.connection.sock, self.reader, self.response.connection.sock)
+        else:
+            asyncio.create_task(self.send(text_data=self.response))
+            asyncio.create_task(self.close(code=1000))
+
+    async def disconnect(self, close_code):
+        if isinstance(self.response, HTTPResponse):
+            loop = asyncio.get_event_loop()
+            loop.remove_reader(self.response.connection.sock)
+            await sync_to_async(self.response.close)()
+        self.conneted = False
+
+
+class AppPodExecConsumer(BaseK8sConsumer):
+
+    async def connect(self):
         await super().connect()
-        self.pod_id = self.scope["url_route"]["kwargs"]["pod_id"]
+        self.stream = None
+        self.conneted = True
+        self.pod_name = self.scope["url_route"]["kwargs"]["name"]
 
-    async def send(self, data):
+    async def send(self, data, channel=STDOUT_CHANNEL):
+        channel_prefix = chr(channel)
         if data is None:
             return
         elif isinstance(data, bytes):
-            await super().send(bytes_data=data)
+            channel_prefix = six.binary_type(channel_prefix, "ascii")
+            await super().send(bytes_data=channel_prefix+data)
         elif isinstance(data, str):
-            await super().send(text_data=data)
+            await super().send(text_data=channel_prefix+data)
+
+    async def wait(self):
+        future, loop = asyncio.Future(), asyncio.get_event_loop()
+        loop.add_reader(self.stream.sock, future.set_result, None)
+        future.add_done_callback(lambda f: loop.remove_reader(self.stream.sock))
+        await future
 
     async def task(self):
-        deadline = time.time() + settings.DRYCC_APP_POD_EXEC_TIMEOUT
-        while self.stream.is_open() and self.conneted and time.time() < deadline:
-            await sync_to_async(self.stream.update)(0.1)
-            if self.stream.peek_stdout():
-                data = self.stream.read_stdout()
-            elif self.stream.peek_stderr():
-                data = self.stream.read_stderr()
-            else:
-                data = None
-            await self.send(data)
-        await self.close(code=1000)
+        try:
+            deadline = time.time() + settings.DRYCC_APP_POD_EXEC_TIMEOUT
+            while self.stream.is_open() and self.conneted and time.time() < deadline:
+                try:
+                    await self.wait()
+                    self.stream.update()
+                    for channel in (ERROR_CHANNEL, STDOUT_CHANNEL, STDERR_CHANNEL):
+                        if channel in self.stream._channels:
+                            data = self.stream.read_channel(channel)
+                            await self.send(data, channel)
+                except ssl.SSLEOFError:
+                    break
+        except exceptions.ApiException as e:
+            await self.send(str(e), STDERR_CHANNEL)
+        finally:
+            await self.close(code=1000)
 
     async def disconnect(self, close_code):
         if self.stream:
-            self.stream.close()
+            await sync_to_async(self.stream.close)()
         self.conneted = False
 
     async def receive(self, text_data=None, bytes_data=None):
         if self.stream is None and text_data is not None:
-            args = (self.kubernetes.connect_get_namespaced_pod_exec, self.pod_id, self.id)
+            args = (self.kubernetes.connect_get_namespaced_pod_exec, self.pod_name, self.id)
             kwargs = json.loads(text_data)
-            kwargs.update({"stderr": True, "stdout": True})
-            if kwargs["stdin"]:
-                kwargs.update({"_preload_content": False})
-                self.stream = stream(*args, **kwargs)
-                asyncio.create_task(self.task())
-            else:
-                await self.send(stream(*args, **kwargs))
-                await self.close(code=1000)
+            kwargs.update({"stderr": True, "stdout": True, "_preload_content": False})
+            self.stream = await sync_to_async(stream)(*args, **kwargs)
+            asyncio.create_task(self.task())
         elif self.stream is not None:
             data = text_data if text_data else bytes_data
             channel, data = ord(data[0]), data[1:]
             await sync_to_async(self.stream.write_channel)(channel, data)
         else:
             raise ValueError("This operation is not supported!")
+
+
+class FilerProxyConsumer(AsyncHttpConsumer):
+    from .middleware import ChannelOAuthMiddleware
+    chunk_size = 64 * 1024
+    middleware = ChannelOAuthMiddleware(None)
+    SKIP_REQUEST_HEADERS = {
+        'host', 'connection', 'keep-alive', 'proxy-connection', 'te', 'trailers', 'upgrade',
+    }
+    SKIP_RESPONSE_HEADERS = {
+        'connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', 'te', 'trailers',
+        'transfer-encoding', 'upgrade',
+    }
+
+    async def handle(self, body: bytes):
+        path = self.scope["url_route"]["kwargs"]["path"]
+        client = await self._get_client(url_path=f"{self.scope["path"].removesuffix(path)}webdav/")
+        if not client:
+            await self.send_response(status=404, body=b'app or volume not found')
+            return
+        if path in ['_/ping', '_/bind']:  # need authentication
+            await self._handle_controller_request(client, path)
+            return
+        elif not path.startswith('webdav/') or (filer := await client.info()) is None:
+            await self.send_response(status=428, body=b'filer service unavailable')
+            return
+        filer_target_url = "{}?{}".format(
+            urljoin(filer["endpoint"], self.scope["path"]), self.scope.get("query_string"))
+        method = self.scope['method'].upper()
+        headers = {
+            name_bytes.decode('latin-1').lower(): value_bytes.decode('latin-1')
+            for name_bytes, value_bytes in self.scope.get('headers', [])
+            if name_bytes.decode('latin-1').lower() not in self.SKIP_REQUEST_HEADERS
+        }
+        await self._handle_proxy_request(filer_target_url, method, headers, body)
+
+    async def _get_client(self, url_path):
+        try:
+            from .filer import FilerClient
+            app = await App.objects.aget(id=self.scope["url_route"]["kwargs"]["id"])
+            volume = await Volume.objects.filter(
+                app=app, name=self.scope["url_route"]["kwargs"]["name"]).afirst()
+            if not volume:
+                return None
+            return FilerClient(app.id, volume, url_path)
+        except App.DoesNotExist:
+            return None
+
+    async def _forward_response(self, response: aiohttp.ClientResponse):
+        response_headers = [
+            [name.encode('latin-1'), value.encode('latin-1')]
+            for name, value in response.headers.items()
+            if name.lower() not in self.SKIP_RESPONSE_HEADERS
+        ]
+        await self.send_headers(status=response.status, headers=response_headers)
+        async for chunk in response.content.iter_chunked(self.chunk_size):
+            if chunk:
+                await self.send_body(chunk, more_body=True)
+        # Send final empty chunk to indicate end of body
+        await self.send_body(b'', more_body=False)
+
+    async def _handle_proxy_request(self, url, method: str, headers: dict[str, str], data: bytes):
+        async with aiohttp.ClientSession(
+            timeout=aiohttp.ClientTimeout(total=settings.DRYCC_FILER_DURATION)
+        ) as session:
+            try:
+                async with session.request(
+                    method=method, url=url, headers=headers, data=data, allow_redirects=False
+                ) as response:
+                    await self._forward_response(response)
+            except aiohttp.ClientError as e:
+                await self.send_response(502, f"proxy service unavailable: {e}".encode('utf-8'))
+            except asyncio.TimeoutError:
+                await self.send_response(504, b'proxy request to backend filer timeout')
+
+    async def _handle_controller_request(self, client, path: str):
+        await self.middleware.login(self.scope)
+        app_perm_checker = AppPermChecker(self.scope)
+        is_ok, message = await app_perm_checker.has_perm()
+        status, body = 200, b''
+        if not is_ok:
+            status, body = 403, message.encode('utf-8')
+        elif path == '_/ping':
+            if (filer := await client.info()) is not None:
+                body = b'pong'
+            else:
+                status, body = 503, b'filer service unavailable'
+        elif path == '_/bind':
+            filer = await client.bind()
+            body = json.dumps({
+                "username": filer["username"], "password": filer["password"],
+            }).encode('utf-8')
+        await self.send_response(status, body)
diff --git a/rootfs/api/exceptions.py b/rootfs/api/exceptions.py
index 0c20a5158..38db7b9b9 100644
--- a/rootfs/api/exceptions.py
+++ b/rootfs/api/exceptions.py
@@ -39,10 +39,11 @@ def custom_exception_handler(exc, context):
     # Call REST framework's default exception handler after specific 404 handling,
     # to get the standard error response.
     response = exception_handler(exc, context)
-
     # No response means DRF couldn't handle it
     # Output a generic 500 in a JSON format
     if response is None:
+        import traceback
+        traceback.print_exc()
         logging.exception('Uncaught Exception', exc_info=exc)
         set_rollback()
         return Response({'detail': 'Server Error'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
diff --git a/rootfs/api/filer.py b/rootfs/api/filer.py
new file mode 100644
index 000000000..5e2d092da
--- /dev/null
+++ b/rootfs/api/filer.py
@@ -0,0 +1,111 @@
+import uuid
+import logging
+import asyncio
+import aiohttp
+from django.conf import settings
+from django.core.cache import cache
+from asgiref.sync import sync_to_async
+
+from .exceptions import DryccException
+from .tasks import send_app_log
+from .utils import random_string, CacheLock, get_scheduler
+
+logger = logging.getLogger(__name__)
+
+
+class FilerClient(object):
+
+    def __init__(self, app_id, volume, url_path):
+        self.path = "/data"
+        self.ports = (9000, 9100)
+        self.app_id = app_id
+        self.volume = volume
+        self.url_path = url_path
+        self.scheduler = get_scheduler()
+
+    def log(self, message, level=logging.INFO):
+        logger.log(level, "[{}]: {}".format(self.app_id, message))
+        send_app_log.delay(self.app_id, message, logging.INFO)
+
+    async def bind(self):
+        lock_key = f"filer:lock:{self.app_id}:{self.volume.name}"
+        lock = CacheLock(lock_key)
+        try:
+            await sync_to_async(lock.acquire)()
+            if _filer := await self.info():
+                return _filer
+            else:
+                _filer = await sync_to_async(self._filer_bind)()
+                await cache.aset(self._cache_key, _filer, timeout=settings.DRYCC_FILER_DURATION)
+        finally:
+            await sync_to_async(lock.release)()
+        return _filer
+
+    async def info(self):
+        _filer = await cache.aget(self._cache_key, None)
+        if _filer and await self._check_health(_filer):
+            await cache.atouch(self._cache_key, timeout=settings.DRYCC_FILER_DURATION)
+            return _filer
+        return None
+
+    @property
+    def _cache_key(self):
+        return f"filer:{self.app_id}:{self.volume.name}"
+
+    def _filer_bind(self):
+        username, password = random_string(32), random_string(32)
+        filer = {"username": username, "password": password}
+        job_ip = self._get_job_ip(self._create_job(username, password))
+        filer.update({
+            "endpoint": f"http://{job_ip}:{self.ports[1]}",
+            "ping_url": f"http://{job_ip}:{self.ports[0]}/_/ping",
+        })
+        return filer
+
+    def _create_job(self, username, password: str) -> str:
+        job_name, k8s_volume = f"drycc-filer-{uuid.uuid4()}", {"name": self.volume.name}
+        if self.volume.type == "csi":
+            k8s_volume.update({"persistentVolumeClaim": {"claimName": self.volume.name}})
+        else:
+            k8s_volume.update(self.volume.parameters)
+        self.scheduler.job.create(self.app_id, job_name, settings.DRYCC_FILER_IMAGE, **{
+            "command": ["init-stack", "/usr/bin/pingguard"],
+            "args": [
+                f"--bind=:{self.ports[0]}", f"--interval={settings.DRYCC_FILER_DURATION}s",
+                "--",
+                "rclone", "serve", "webdav", self.path, f"--addr=:{self.ports[1]}",
+                f"--user={username}", f"--pass={password}", f"--baseurl={self.url_path}",
+            ],
+            "app_type": "filer", "replicas": 1, "deploy_timeout": 120, "volumes": [k8s_volume],
+            "volume_mounts": [{"mountPath": self.path, "name": self.volume.name}],
+            "restart_policy": "Never", "image_pull_policy": settings.DRYCC_FILER_IMAGE_PULL_POLICY,
+            "pod_security_context": {"fsGroup": 1001, "runAsGroup": 1001, "runAsUser": 1001},
+            "active_deadline_seconds": 2 ** 32,
+            "ttl_seconds_after_finished": settings.DRYCC_FILER_DURATION,
+        })
+        return job_name
+
+    def _get_job_ip(self, job_name: str) -> str:
+        state, labels = 'initializing', {'job-name': job_name}
+        for count, state in enumerate(self.scheduler.pod.watch(
+            self.app_id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT,
+            until_states=['up', 'down', 'crashed', 'error'],
+        )):
+            self.log(f"waiting for filer bind: {state} * {count}")
+        if state != 'up':
+            raise DryccException(f'filer startup failed, current status: {state}')
+        pods = self.scheduler.pod.get(self.app_id, labels=labels).json()
+        if not pods["items"]:
+            raise DryccException('filer pod not found after run completed')
+        return pods["items"][0]["status"]["podIP"]
+
+    async def _check_health(self, filer):
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(
+                    filer["ping_url"],
+                    timeout=aiohttp.ClientTimeout(total=5),
+                ) as response:
+                    return response.status == 200
+        except (aiohttp.ClientError, asyncio.TimeoutError):
+            return False
diff --git a/rootfs/api/schemas/__init__.py b/rootfs/api/files/__init__.py
similarity index 100%
rename from rootfs/api/schemas/__init__.py
rename to rootfs/api/files/__init__.py
diff --git a/rootfs/api/files/parsers.py b/rootfs/api/files/parsers.py
new file mode 100644
index 000000000..867b943c3
--- /dev/null
+++ b/rootfs/api/files/parsers.py
@@ -0,0 +1,58 @@
+import contextlib
+from django.utils.http import parse_header_parameters
+from django.core.files.uploadedfile import UploadedFile
+from rest_framework.exceptions import ParseError
+from rest_framework.parsers import FileUploadParser, DataAndFiles
+
+
+class FilerFile(UploadedFile):
+
+    def __init__(self, filename, filepath, input_data, content_length):
+        self.filename = filename
+        self.filepath = filepath
+        super().__init__(input_data, name=filename, size=content_length)
+
+
+class FilerUploadParser(FileUploadParser):
+    """
+    Filer upload parser.
+    """
+    media_type = 'filer/octet-stream'
+
+    errors = {
+        "no_filepath": (
+            "Missing filepath. Request should include a Content-Disposition "
+            "header with a filepath parameter."
+        ),
+    }
+    errors.update(FileUploadParser.errors)
+
+    def parse(self, stream, media_type=None, parser_context=None):
+        request = parser_context['request']
+        filename = self.get_filename(stream, media_type, parser_context)
+        if not filename:
+            raise ParseError(self.errors['no_filename'])
+        filepath = self.get_filepath(stream, media_type, parser_context)
+        if not filepath:
+            raise ParseError(self.errors['no_filepath'])
+        try:
+            content_length = int(request.META.get('HTTP_CONTENT_LENGTH',
+                                                  request.META.get('CONTENT_LENGTH', 0)))
+        except (ValueError, TypeError):
+            content_length = None
+        return DataAndFiles({}, {'file': FilerFile(filename, filepath, stream, content_length)})
+
+    def get_filepath(self, stream, media_type, parser_context):
+        """
+        Detects the uploaded file name. First searches a 'filepath' url kwarg.
+        Then tries to parse Content-Disposition header.
+        """
+        with contextlib.suppress(KeyError):
+            return parser_context['kwargs']['filepath']
+
+        with contextlib.suppress(AttributeError, KeyError, ValueError):
+            meta = parser_context['request'].META
+            _, params = parse_header_parameters(meta['HTTP_CONTENT_DISPOSITION'])
+            if 'filepath*' in params:
+                return params['filepath*']
+            return params['filepath']
diff --git a/rootfs/api/fixtures/tests.json b/rootfs/api/fixtures/tests.json
index 94b6182ab..7ac08f75e 100644
--- a/rootfs/api/fixtures/tests.json
+++ b/rootfs/api/fixtures/tests.json
@@ -88,5 +88,243 @@
         "email": "autotest4@drycc.cc",
         "date_joined": "2013-05-10T16:08:09.357Z"
     }
+},
+{
+  "model": "api.limitspec",
+  "pk": "std1",
+  "fields": {
+    "cpu": {
+      "name": "Universal CPU",
+      "cores": 32,
+      "clock": "3100MHZ",
+      "boost": "3700MHZ",
+      "threads": 64
+    },
+    "memory": {
+      "size": "64GB",
+      "type": "DDR4-ECC"
+    },
+    "features": {
+      "gpu": {
+        "name": "Integrated GPU",
+        "tmus": 1,
+        "rops": 1,
+        "cores": 128,
+        "memory": {
+          "size": "shared",
+          "type": "shared"
+        }
+      },
+      "network": "10G"
+    },
+    "keywords": ["amd", "intel", "unknown"],
+    "disabled": false,
+    "created": "2024-03-13T10:49:49.773Z",
+    "updated": "2024-03-13T10:49:49.773Z"
+  }
+},
+{
+  "model": "api.limitplan",
+  "pk": "std1.large.c1m1",
+  "fields": {
+    "spec_id": "std1",
+    "cpu": 1,
+    "memory": 1,
+    "features": {
+      "gpu": 1,
+      "network": 1
+    },
+    "disabled": false,
+    "priority": 100,
+    "limits": {
+      "cpu": 1,
+      "memory": "1Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "requests": {
+      "cpu": 0.5,
+      "memory": "0.5Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "annotations": {
+      "kubernetes.io/egress-bandwidth": "10M",
+      "kubernetes.io/ingress-bandwidth": "10M"
+    },
+    "node_selector": {},
+    "pod_security_context": {},
+    "container_security_context": {},
+    "created": "2024-03-13T11:08:07.681Z",
+    "updated": "2024-03-13T11:08:07.681Z"
+  }
+},
+{
+  "model": "api.limitplan",
+  "pk": "std1.large.c1m2",
+  "fields": {
+    "spec_id": "std1",
+    "cpu": 1,
+    "memory": 2,
+    "features": {
+      "gpu": 1,
+      "network": 1
+    },
+    "disabled": false,
+    "priority": 101,
+    "limits": {
+      "cpu": 1,
+      "memory": "2Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "requests": {
+      "cpu": 0.5,
+      "memory": "1Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "annotations": {
+      "kubernetes.io/egress-bandwidth": "10M",
+      "kubernetes.io/ingress-bandwidth": "10M"
+    },
+    "node_selector": {},
+    "pod_security_context": {},
+    "container_security_context": {},
+    "created": "2024-03-13T11:08:07.681Z",
+    "updated": "2024-03-13T11:08:07.681Z"
+  }
+},
+{
+  "model": "api.limitplan",
+  "pk": "std1.large.c1m4",
+  "fields": {
+    "spec_id": "std1",
+    "cpu": 1,
+    "memory": 4,
+    "features": {
+      "gpu": 1,
+      "network": 1
+    },
+    "disabled": false,
+    "priority": 102,
+    "limits": {
+      "cpu": 1,
+      "memory": "4Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "requests": {
+      "cpu": 0.5,
+      "memory": "2Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "annotations": {
+      "kubernetes.io/egress-bandwidth": "20M",
+      "kubernetes.io/ingress-bandwidth": "20M"
+    },
+    "node_selector": {},
+    "pod_security_context": {},
+    "container_security_context": {},
+    "created": "2024-03-13T11:08:07.681Z",
+    "updated": "2024-03-13T11:08:07.681Z"
+  }
+},
+{
+  "model": "api.limitplan",
+  "pk": "std1.large.c2m2",
+  "fields": {
+    "spec_id": "std1",
+    "cpu": 2,
+    "memory": 2,
+    "features": {
+      "gpu": 1,
+      "network": 1
+    },
+    "disabled": false,
+    "priority": 103,
+    "limits": {
+      "cpu": 2,
+      "memory": "2Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "requests": {
+      "cpu": 1,
+      "memory": "1Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "annotations": {
+      "kubernetes.io/egress-bandwidth": "20M",
+      "kubernetes.io/ingress-bandwidth": "20M"
+    },
+    "node_selector": {},
+    "pod_security_context": {},
+    "container_security_context": {},
+    "created": "2024-03-13T11:08:07.681Z",
+    "updated": "2024-03-13T11:08:07.681Z"
+  }
+},
+{
+  "model": "api.limitplan",
+  "pk": "std1.large.c2m4",
+  "fields": {
+    "spec_id": "std1",
+    "cpu": 2,
+    "memory": 4,
+    "features": {
+      "gpu": 1,
+      "network": 1
+    },
+    "disabled": false,
+    "priority": 104,
+    "limits": {
+      "cpu": 2,
+      "memory": "4Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "requests": {
+      "cpu": 1,
+      "memory": "2Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "annotations": {
+      "kubernetes.io/egress-bandwidth": "30M",
+      "kubernetes.io/ingress-bandwidth": "30M"
+    },
+    "node_selector": {},
+    "pod_security_context": {},
+    "container_security_context": {},
+    "created": "2024-03-13T11:08:07.681Z",
+    "updated": "2024-03-13T11:08:07.681Z"
+  }
+},
+{
+  "model": "api.limitplan",
+  "pk": "std1.large.c2m8",
+  "fields": {
+    "spec_id": "std1",
+    "cpu": 2,
+    "memory": 8,
+    "features": {
+      "gpu": 1,
+      "network": 1
+    },
+    "disabled": false,
+    "priority": 105,
+    "limits": {
+      "cpu": 2,
+      "memory": "8Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "requests": {
+      "cpu": 1,
+      "memory": "4Gi",
+      "ephemeral-storage": "2Gi"
+    },
+    "annotations": {
+      "kubernetes.io/egress-bandwidth": "30M",
+      "kubernetes.io/ingress-bandwidth": "30M"
+    },
+    "node_selector": {},
+    "pod_security_context": {},
+    "container_security_context": {},
+    "created": "2024-03-13T11:08:07.681Z",
+    "updated": "2024-03-13T11:08:07.681Z"
+  }
 }
 ]
diff --git a/rootfs/api/management/commands/cluster_migrate.py b/rootfs/api/management/commands/cluster_migrate.py
deleted file mode 100644
index f7110b23b..000000000
--- a/rootfs/api/management/commands/cluster_migrate.py
+++ /dev/null
@@ -1,11 +0,0 @@
-from django.core.cache import cache
-from django.core.management.commands import migrate
-
-
-class Command(migrate.Command):
-    """Management command for cluster_lock"""
-    def handle(self, *args, **options):
-        print("preparing migrate...")
-        with cache.lock("drycc:controller", timeout=60 * 60 * 24):
-            super().handle(self, *args, **options)
-        print("migrate completed...")
diff --git a/rootfs/api/management/commands/load_db_state_to_k8s.py b/rootfs/api/management/commands/load_db_state_to_k8s.py
index b98ebecdf..ebce06e61 100644
--- a/rootfs/api/management/commands/load_db_state_to_k8s.py
+++ b/rootfs/api/management/commands/load_db_state_to_k8s.py
@@ -7,6 +7,8 @@
 from api.models.domain import Domain
 from api.models.certificate import Certificate
 from api.models.service import Service
+from api.models.volume import Volume
+from api.models.gateway import Route, Gateway
 from api.exceptions import DryccException, AlreadyExists
 
 
@@ -18,14 +20,9 @@ class Command(BaseCommand):
     to k8s.
     """
 
-    def _deploy(self, app, release):
-        if release.build is None:
-            print('WARNING: {} has no build associated with '
-                  'its latest release. Skipping deployment...'.format(app))
-            return
-
+    def _deploy(self, app):
         try:
-            app.deploy(release)
+            app.state_to_k8s()
         except AlreadyExists as error:
             logger.debug(error)
             print('WARNING: {} has a deployment in progress. '
@@ -48,13 +45,9 @@ def handle(self, *args, **options):
                 cert.attach_in_kubernetes(domain)
 
         # deploy apps
-        print("Deploying available applications")
+        print("Deploying available applications.")
         for app in App.objects.all():
-            release = app.release_set.filter(failed=False).latest()
-            if release.canary:
-                self._deploy(app, app.release_set.filter(failed=False, canary=False).latest())
-            self._deploy(app, release)
-
+            self._deploy(app)
         print("Done Publishing DB state to kubernetes.")
 
     def save_apps(self):
@@ -63,11 +56,13 @@ def save_apps(self):
             try:
                 app.save()
                 app.config_set.latest().save()
-                app.tls_set.latest().sync()
+                tls = app.tls_set.latest()
+                tls.refresh_issuer_to_k8s()
+                tls.refresh_certificate_to_k8s()
             except DryccException as error:
                 print('ERROR: Problem saving to model {} for {}'
                       'due to {}'.format(str(App.__name__), str(app), str(error)))
-        for model in (Key, Domain, Certificate, Service):
+        for model in (Volume, Route, Gateway, Key, Domain, Certificate, Service):
             for obj in model.objects.all():
                 try:
                     obj.save()
diff --git a/rootfs/api/management/commands/measure_apps.py b/rootfs/api/management/commands/measure_apps.py
deleted file mode 100644
index 8b677c361..000000000
--- a/rootfs/api/management/commands/measure_apps.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import uuid
-import time
-import logging
-from django.utils import timezone
-from django.core.management.base import BaseCommand
-from django.conf import settings
-from api.models.app import App
-from api.tasks import send_measurements
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Management command for push data to manager"""
-
-    def handle(self, *args, **options):
-        if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
-            app_list = []
-            for app in App.objects.all():
-                app_list.extend(app.to_measurements(timestamp))
-                if len(app_list) % 1000 == 0:
-                    send_measurements.delay(app_list)
-                    app_list = []
-            if len(app_list) > 0:
-                send_measurements.delay(app_list)
-            logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
-            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_loadbalancers.py b/rootfs/api/management/commands/measure_loadbalancers.py
deleted file mode 100644
index b6ad3853d..000000000
--- a/rootfs/api/management/commands/measure_loadbalancers.py
+++ /dev/null
@@ -1,50 +0,0 @@
-import uuid
-import time
-import logging
-from django.utils import timezone
-from django.core.management.base import BaseCommand
-from django.conf import settings
-from api import monitor
-from api.models.app import App
-from api.tasks import send_measurements
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Management command for push data to manager"""
-
-    def _measure_loadbalancers(self, app_map, timestamp):
-        stop = timestamp - (timestamp % 3600)
-        start = stop - 3600
-        loadbalancers = []
-        for item in monitor.query_loadbalancer(app_map.keys(), start, stop):
-            name = item["ip"]
-            namespace = item["namespace"]
-            owner_id = app_map[namespace].owner_id
-            loadbalancers.append({
-                "app_id":  str(app_map[namespace].uuid),
-                "owner": owner_id,
-                "name": name,
-                "type": "loadbalancer",
-                "unit": "number",
-                "usage": 1,
-                "timestamp": start
-            })
-        send_measurements.delay(loadbalancers)
-
-    def handle(self, *args, **options):
-        if settings.WORKFLOW_MANAGER_URL and settings.DRYCC_PROMETHEUS_URL:
-            timestamp = int(time.time())
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
-            app_map = {}
-            for app in App.objects.all():
-                app_map[app.id] = app
-                if len(app_map) % 1000 == 0:
-                    self._measure_loadbalancers(app_map, timestamp)
-                    app_map = {}
-            if len(app_map) > 0:
-                self._measure_loadbalancers(app_map, timestamp)
-            logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
-            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_networks.py b/rootfs/api/management/commands/measure_networks.py
deleted file mode 100644
index c38c62be3..000000000
--- a/rootfs/api/management/commands/measure_networks.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import uuid
-import time
-import logging
-from django.utils import timezone
-from django.core.management.base import BaseCommand
-from django.conf import settings
-from api import monitor
-from api.models.app import App
-from api.tasks import send_measurements
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Management command for push data to manager"""
-
-    def _measure_networks(self, app_map, timestamp):
-        stop = timestamp - (timestamp % 3600)
-        start = stop - 3600
-        networks = []
-        for namespace, pod_name, rx_bytes, tx_bytes in monitor.query_network_flow(
-                app_map.keys(), start, stop):
-            owner_id = app_map[namespace].owner_id
-            networks.append({
-                "app_id":  str(app_map[namespace].uuid),
-                "owner": owner_id,
-                "name": pod_name,
-                "type": "network",
-                "unit": "bytes",
-                "usage": rx_bytes + tx_bytes,
-                "timestamp": start
-            })
-        send_measurements.delay(networks)
-
-    def handle(self, *args, **options):
-        if settings.WORKFLOW_MANAGER_URL:
-            timestamp = int(time.time())
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} limits to workflow_manager when {timezone.now()}")
-            app_map = {}
-            for app in App.objects.all():
-                app_map[app.id] = app
-                if len(app_map) % 1000 == 0:
-                    self._measure_networks(app_map, timestamp)
-                    app_map = {}
-            if len(app_map) > 0:
-                self._measure_networks(app_map, timestamp)
-            logger.info(f"pushed {task_id} limits to workflow_manager when {timezone.now()}")
-            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_volumes.py b/rootfs/api/management/commands/measure_volumes.py
deleted file mode 100644
index ecf14c524..000000000
--- a/rootfs/api/management/commands/measure_volumes.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import uuid
-import time
-import logging
-from django.utils import timezone
-from django.core.management.base import BaseCommand
-from django.conf import settings
-from api.models.volume import Volume
-from api.tasks import send_measurements
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Management command for push data to manager"""
-
-    def handle(self, *args, **options):
-        if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} volumes to workflow_manager when {timezone.now()}")
-            volume_list = []
-            for volume in Volume.objects.all():
-                volume_list.extend(volume.to_measurements(timestamp))
-                if len(volume_list) % 1000 == 0:
-                    send_measurements.delay(volume_list)
-                    volume_list = []
-            if len(volume_list) > 0:
-                send_measurements.delay(volume_list)
-            logger.info(f"pushed {task_id} volumes to workflow_manager when {timezone.now()}")
-            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_app_usage.py b/rootfs/api/management/commands/upload_app_usage.py
new file mode 100644
index 000000000..82028125c
--- /dev/null
+++ b/rootfs/api/management/commands/upload_app_usage.py
@@ -0,0 +1,39 @@
+import uuid
+import logging
+import random
+from datetime import timedelta
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from api.models.app import App
+from api.tasks import send_usage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Management command for push data to manager"""
+
+    def handle(self, *args, **options):
+        if settings.WORKFLOW_MANAGER_URL:
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} resources to workflow_manager when {now}")
+            app_list = []
+            for app in App.objects.all():
+                app_list.extend(app.to_usages(timestamp))
+                if len(app_list) % 1000 == 0:
+                    logger.info(f"bulk sent: {len(app_list)}")
+                    send_usage.apply_async(
+                        args=(app_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 3600))
+                    )
+                    app_list = []
+            if len(app_list) > 0:
+                logger.info(f"bulk sent: {len(app_list)}")
+                send_usage.apply_async(
+                    args=(app_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 3600))
+                )
+            logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
+            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_gateway_usage.py b/rootfs/api/management/commands/upload_gateway_usage.py
new file mode 100644
index 000000000..aaf9b8ba6
--- /dev/null
+++ b/rootfs/api/management/commands/upload_gateway_usage.py
@@ -0,0 +1,39 @@
+import uuid
+import logging
+import random
+from datetime import timedelta
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from api.models.gateway import Gateway
+from api.tasks import send_usage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Management command for push data to manager"""
+
+    def handle(self, *args, **options):
+        if settings.WORKFLOW_MANAGER_URL:
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} gateways to workflow_manager when {now}")
+            gateway_list = []
+            for gateway in Gateway.objects.all():
+                gateway_list.extend(gateway.to_usages(timestamp))
+                if len(gateway_list) % 1000 == 0:
+                    logger.info(f"bulk sent: {len(gateway_list)}")
+                    send_usage.apply_async(
+                        args=(gateway_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 3600))
+                    )
+                    gateway_list = []
+            if len(gateway_list) > 0:
+                logger.info(f"bulk sent: {len(gateway_list)}")
+                send_usage.apply_async(
+                    args=(gateway_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 3600))
+                )
+            logger.info(f"pushed {task_id} gateways to workflow_manager when {timezone.now()}")
+            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_network_usage.py b/rootfs/api/management/commands/upload_network_usage.py
new file mode 100644
index 000000000..fd80ea387
--- /dev/null
+++ b/rootfs/api/management/commands/upload_network_usage.py
@@ -0,0 +1,60 @@
+import uuid
+import hashlib
+import logging
+import random
+from datetime import timedelta
+from asgiref.sync import async_to_sync
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from api import monitor
+from api.models.app import App
+from api.tasks import send_usage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Management command for push data to manager"""
+
+    def _upload_network_usage(self, start_time, app_map, timestamp):
+        stop = timestamp - (timestamp % 3600)
+        start = stop - 3600
+        networks = []
+        for item in async_to_sync(monitor.query_network_usage)(app_map.keys(), start, stop):
+            metric = item["metric"]
+            _, value = item["value"]
+            pod_name = metric['pod']
+            networks.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "workspace": app_map[metric['namespace']].workspace_id,
+                "type": "network",
+                "unit": "bytes",
+                "name": metric['direction'],
+                "usage": value,
+                "kwargs": {
+                    "pod": pod_name,
+                },
+                "timestamp": start,
+                "identifier": hashlib.md5(pod_name.encode("utf-8")).hexdigest(),
+            })
+        logger.info(f"bulk sent: {len(networks)}")
+        send_usage.apply_async(
+            args=(networks,), eta=start_time + timedelta(seconds=random.randint(1, 3600))
+        )
+
+    def handle(self, *args, **options):
+        if settings.WORKFLOW_MANAGER_URL:
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} networks to workflow_manager when {now}")
+            app_map = {}
+            for app in App.objects.all():
+                app_map[app.id] = app
+                if len(app_map) % 1000 == 0:
+                    self._upload_network_usage(start_time, app_map, timestamp)
+                    app_map = {}
+            if len(app_map) > 0:
+                self._upload_network_usage(start_time, app_map, timestamp)
+            logger.info(f"pushed {task_id} networks to workflow_manager when {timezone.now()}")
+            self.stdout.write("done")
diff --git a/rootfs/api/management/commands/measure_resources.py b/rootfs/api/management/commands/upload_resource_usage.py
similarity index 50%
rename from rootfs/api/management/commands/measure_resources.py
rename to rootfs/api/management/commands/upload_resource_usage.py
index b1f9ceccb..e9b598912 100644
--- a/rootfs/api/management/commands/measure_resources.py
+++ b/rootfs/api/management/commands/upload_resource_usage.py
@@ -1,11 +1,12 @@
 import uuid
-import time
 import logging
+import random
+from datetime import timedelta
 from django.utils import timezone
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from api.models.resource import Resource
-from api.tasks import send_measurements
+from api.tasks import send_usage
 
 logger = logging.getLogger(__name__)
 
@@ -15,16 +16,24 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         if settings.WORKFLOW_MANAGER_URL:
-            timestamp = time.time()
-            task_id = uuid.uuid4().hex
-            logger.info(f"pushing {task_id} resources to workflow_manager when {timezone.now()}")
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} resources to workflow_manager when {now}")
             resource_list = []
             for resource in Resource.objects.filter(status="Ready"):
-                resource_list.extend(resource.to_measurements(timestamp))
+                resource_list.extend(resource.to_usages(timestamp))
                 if len(resource_list) % 1000 == 0:
-                    send_measurements.delay(resource_list)
+                    logger.info(f"bulk sent: {len(resource_list)}")
+                    send_usage.apply_async(
+                        args=(resource_list,),
+                        eta=start_time + timedelta(seconds=random.randint(1, 3600))
+                    )
                     resource_list = []
             if len(resource_list) > 0:
-                send_measurements.delay(resource_list)
+                logger.info(f"bulk sent: {len(resource_list)}")
+                send_usage.apply_async(
+                    args=(resource_list,),
+                    eta=start_time + timedelta(seconds=random.randint(1, 3600))
+                )
             logger.info(f"pushed {task_id} resources to workflow_manager when {timezone.now()}")
             self.stdout.write("done")
diff --git a/rootfs/api/management/commands/upload_volume_usage.py b/rootfs/api/management/commands/upload_volume_usage.py
new file mode 100644
index 000000000..9852e42ff
--- /dev/null
+++ b/rootfs/api/management/commands/upload_volume_usage.py
@@ -0,0 +1,60 @@
+import uuid
+import hashlib
+import logging
+import random
+from datetime import timedelta
+from asgiref.sync import async_to_sync
+from django.utils import timezone
+from django.core.management.base import BaseCommand
+from django.conf import settings
+from api import monitor
+from api.models.app import App
+from api.tasks import send_usage
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Management command for push data to manager"""
+
+    def _upload_volume_usage(self, start_time, app_map, timestamp):
+        stop = timestamp - (timestamp % 3600)
+        start = stop - 3600
+        volumes = []
+        for item in async_to_sync(monitor.query_volume_usage)(app_map.keys(), start, stop):
+            metric = item["metric"]
+            _, value = item["value"]
+            pvc_name = metric['persistentvolumeclaim']
+            volumes.append({
+                "app_id":  str(app_map[metric['namespace']].uuid),
+                "workspace": app_map[metric['namespace']].workspace_id,
+                "type": "volume",
+                "unit": "bytes",
+                "name": metric["storageclass"],
+                "usage": value,
+                "kwargs": {
+                    "persistentvolumeclaim": pvc_name,
+                },
+                "timestamp": start,
+                "identifier": hashlib.md5(pvc_name.encode("utf-8")).hexdigest(),
+            })
+        logger.info(f"bulk sent: {len(volumes)}")
+        send_usage.apply_async(
+            args=(volumes,), eta=start_time + timedelta(seconds=random.randint(1, 3600))
+        )
+
+    def handle(self, *args, **options):
+        if settings.WORKFLOW_MANAGER_URL:
+            now = timezone.now()
+            start_time, timestamp, task_id = now, int(now.timestamp()), uuid.uuid4().hex
+            logger.info(f"pushing {task_id} volumes to workflow_manager when {now}")
+            app_map = {}
+            for app in App.objects.all():
+                app_map[app.id] = app
+                if len(app_map) % 1000 == 0:
+                    self._upload_volume_usage(start_time, app_map, timestamp)
+                    app_map = {}
+            if len(app_map) > 0:
+                self._upload_volume_usage(start_time, app_map, timestamp)
+            logger.info(f"pushed {task_id} volumes to workflow_manager when {timezone.now()}")
+            self.stdout.write("done")
diff --git a/rootfs/api/manager.py b/rootfs/api/manager.py
index 18a7b2ebb..53b22eb7f 100644
--- a/rootfs/api/manager.py
+++ b/rootfs/api/manager.py
@@ -1,14 +1,19 @@
 import base64
+import logging
 import requests
 from typing import List, Dict
+from django.core.cache import cache
 from requests_toolbelt import user_agent
 from django.conf import settings
 from api import __version__ as drycc_version
 
+logger = logging.getLogger(__name__)
+
 
 class ManagerAPI(object):
 
-    def __init__(self):
+    def __init__(self, timeout=3):
+        self.timeout = timeout
         token = base64.b85encode(b"%s:%s" % (
             settings.WORKFLOW_MANAGER_ACCESS_KEY.encode("utf8"),
             settings.WORKFLOW_MANAGER_SECRET_KEY.encode("utf8"),
@@ -23,6 +28,7 @@ def request(self, method, url, **kwargs):
         headers = kwargs.get("headers", {})
         headers.update(self.headers)
         kwargs["headers"] = headers
+        kwargs["timeout"] = self.timeout
         return requests.request(method, url, **kwargs)
 
     def get(self, url, params=None, **kwargs):
@@ -48,34 +54,49 @@ def delete(self, url, **kwargs):
         return self.request('delete', url, **kwargs)
 
 
-class User(ManagerAPI):
+class WorkspaceAPI(ManagerAPI):
 
-    def get_status(self, id):
+    def get_status(self, workspace_id):
         """
         {
             "is_active": False,
             "message": "The user is in arrears"
         }
         """
-        url = f"{settings.WORKFLOW_MANAGER_URL}/users/{id}/status/"
-        return self.get(url=url).json()
+        key = f"workspace:status:{workspace_id}"
+        status = cache.get(key)
+        if not status:
+            url = f"{settings.WORKFLOW_MANAGER_URL}/workspaces/{workspace_id}/status/"
+            try:
+                status = self.get(url=url, timeout=self.timeout).json()
+            except requests.exceptions.Timeout as ex:
+                msg = f"request workspace {workspace_id} timeout, skipping verification."
+                status = {"is_active": True, "message": msg}
+                logger.error(msg)
+                logger.exception(ex)
+            cache.set(key, status, timeout=settings.DRYCC_CACHE_USER_TIME)
+        return status
+
+
+class UserAPI(WorkspaceAPI):
+    """Backward-compatible alias for legacy call sites."""
 
 
-class Measurement(ManagerAPI):
+class UsageAPI(ManagerAPI):
 
-    def post(self, measurements: List[Dict[str, str]]):
+    def post(self, usages: List[Dict[str, str]]):
         """
         [
             {
                 "app_id":  "test",
-                "owner": "test",
-                "name": web,
-                "type": "CPU",
-                "unit": "G"
+                "workspace": "test",
+                "name": "web",
+                "type": "limits",
+                "unit": "std1.large.c1m1",
                 "usage": "2",
                 "timestamp": "1609231998.9103732"
             }
         ]
         """
-        url = "%s/measurements/" % settings.WORKFLOW_MANAGER_URL
-        return super().post(url=url, json=measurements)
+        url = "%s/usages/" % settings.WORKFLOW_MANAGER_URL
+        return super().post(url=url, json=usages)
diff --git a/rootfs/api/middleware.py b/rootfs/api/middleware.py
index 651f60982..47cc095be 100644
--- a/rootfs/api/middleware.py
+++ b/rootfs/api/middleware.py
@@ -3,15 +3,18 @@
 
 See https://docs.djangoproject.com/en/1.11/topics/http/middleware/
 """
-import collections
-from api import __version__
+
+from django.http.request import HttpRequest
 
 from channels.middleware import BaseMiddleware
 from channels.db import database_sync_to_async
-from api.authentication import DryccAuthentication
+
+from api import __version__
+
+from .authentication import DryccAuthentication
 
 
-class APIVersionMiddleware(object):
+class DjangoAPIVersionMiddleware(object):
     """
     Include that REST API version with each response.
     """
@@ -32,25 +35,48 @@ def __call__(self, request):
         return response
 
 
+class ChannelAPIVersionMiddleware(BaseMiddleware):
+    """
+    Channel middleware to include REST API version with each HTTP response.
+    """
+    async def __call__(self, scope, receive, send):
+        if scope["type"] == "http":
+            async def send_with_version(message):
+                if message["type"] == "http.response.start":
+                    headers = list(message.get("headers", []))
+                    # clients shouldn't care about the patch release
+                    version = __version__.rsplit('.', 1)[0]
+                    headers.append((b"DRYCC_API_VERSION", version.encode()))
+                    headers.append((b"DRYCC_PLATFORM_VERSION", __version__.encode()))
+                    message = message.copy()
+                    message["headers"] = headers
+                await send(message)
+            return await super().__call__(scope, receive, send_with_version)
+        else:
+            return await super().__call__(scope, receive, send)
+
+
 class ChannelOAuthMiddleware(BaseMiddleware):
     """
     Middleware which populates scope["user"] from a auth2 token.
     """
-    Request = collections.namedtuple('Request', ["META"])
+    authentication = DryccAuthentication()
 
-    async def get_user(self, scope):
+    @database_sync_to_async
+    def login(self, scope):
         headers = {}
         for header in scope["headers"]:
-            if header[0] in (b"user-agent", b"authorization"):
+            if header[0] in (b"authorization", ):
                 key = "HTTP_%s" % header[0].decode().replace("-", "_").upper()
                 headers[key] = header[1].decode()
-        if len(headers) != 2:
+        if len(headers) < 1:
             return None
-        request = self.Request(headers)
-        user, _ = await database_sync_to_async(DryccAuthentication().authenticate)(request)
-        return user
+        request = HttpRequest()
+        request.META = headers
+        scope["user"], _ = self.authentication.authenticate(request)
+        return scope["user"]
 
     async def __call__(self, scope, receive, send):
         scope = dict(scope)
-        scope["user"] = await self.get_user(scope)
+        await self.login(scope)
         return await super().__call__(scope, receive, send)
diff --git a/rootfs/api/migrations/0002_volume_parameters_volume_type_alter_volume_path_and_more.py b/rootfs/api/migrations/0002_volume_parameters_volume_type_alter_volume_path_and_more.py
new file mode 100644
index 000000000..8fcd8f55a
--- /dev/null
+++ b/rootfs/api/migrations/0002_volume_parameters_volume_type_alter_volume_path_and_more.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.10 on 2024-02-23 03:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='volume',
+            name='parameters',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AddField(
+            model_name='volume',
+            name='type',
+            field=models.CharField(choices=[('csi', 'container storage interface'), ('nfs', 'network file system')], default='csi'),
+        ),
+        migrations.AlterField(
+            model_name='volume',
+            name='path',
+            field=models.JSONField(default=dict),
+        ),
+        migrations.AlterField(
+            model_name='volume',
+            name='size',
+            field=models.CharField(default='0G', max_length=128),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0003_limitspec_remove_app_procfile_structure_and_more.py b/rootfs/api/migrations/0003_limitspec_remove_app_procfile_structure_and_more.py
new file mode 100644
index 000000000..d05784775
--- /dev/null
+++ b/rootfs/api/migrations/0003_limitspec_remove_app_procfile_structure_and_more.py
@@ -0,0 +1,75 @@
+# Generated by Django 4.2.10 on 2024-03-18 06:42
+
+import api.models.limit
+from django.db import migrations, models
+import django.db.models.deletion
+import functools
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0002_volume_parameters_volume_type_alter_volume_path_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LimitSpec',
+            fields=[
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('id', models.CharField(max_length=63, primary_key=True, serialize=False)),
+                ('cpu', models.JSONField(validators=[functools.partial(api.models.limit.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'properties': {'boost': {'type': 'string'}, 'clock': {'type': 'string'}, 'cores': {'type': 'integer'}, 'name': {'type': 'string'}, 'threads': {'type': 'integer'}}, 'required': ['name', 'cores', 'clock', 'boost', 'threads'], 'type': 'object'}})])),
+                ('memory', models.JSONField(validators=[functools.partial(api.models.limit.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'properties': {'size': {'type': 'string'}, 'type': {'type': 'string'}}, 'required': ['size', 'type'], 'type': 'object'}})])),
+                ('features', models.JSONField(validators=[functools.partial(api.models.limit.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'properties': {'gpu': {'$schema': 'http://json-schema.org/schema#', 'properties': {'cores': {'type': 'integer'}, 'memory': {'$schema': 'http://json-schema.org/schema#', 'properties': {'size': {'type': 'string'}, 'type': {'type': 'string'}}, 'required': ['size', 'type'], 'type': 'object'}, 'name': {'type': 'string'}, 'rops': {'type': 'integer'}, 'tmus': {'type': 'integer'}}, 'required': ['name', 'tmus', 'rops', 'cores'], 'type': 'object'}, 'network': {'type': 'string'}}, 'required': ['gpu', 'network'], 'type': 'object'}})])),
+                ('keywords', models.JSONField(validators=[functools.partial(api.models.limit.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'items': {'type': 'string'}, 'minItems': 1, 'type': 'array'}})])),
+                ('disabled', models.BooleanField(default=False)),
+                ('priority', models.SmallIntegerField(default=100)),
+            ],
+            options={
+                'ordering': ['-priority'],
+                'get_latest_by': 'created',
+            },
+        ),
+        migrations.RemoveField(
+            model_name='app',
+            name='procfile_structure',
+        ),
+        migrations.RemoveField(
+            model_name='config',
+            name='cpu',
+        ),
+        migrations.RemoveField(
+            model_name='config',
+            name='memory',
+        ),
+        migrations.AddField(
+            model_name='config',
+            name='limits',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.CreateModel(
+            name='LimitPlan',
+            fields=[
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('id', models.CharField(max_length=63, primary_key=True, serialize=False)),
+                ('cpu', models.SmallIntegerField(default=1)),
+                ('memory', models.SmallIntegerField(default=1)),
+                ('features', models.JSONField(validators=[functools.partial(api.models.limit.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'properties': {'gpu': {'type': 'integer'}, 'network': {'type': 'integer'}}, 'required': ['gpu', 'memory'], 'type': 'object'}})])),
+                ('disabled', models.BooleanField(default=False)),
+                ('priority', models.SmallIntegerField(default=100)),
+                ('limits', models.JSONField(validators=[functools.partial(api.models.limit.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'properties': {'cpu': {'type': 'integer'}, 'memory': {'type': 'integer'}}, 'required': ['cpu', 'memory'], 'type': 'object'}})])),
+                ('requests', models.JSONField(default=dict)),
+                ('annotations', models.JSONField(default=dict)),
+                ('node_selector', models.JSONField(default=dict)),
+                ('pod_security_context', models.JSONField(default=dict)),
+                ('container_security_context', models.JSONField(default=dict)),
+                ('spec', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='api.limitspec')),
+            ],
+            options={
+                'ordering': ['priority'],
+                'get_latest_by': 'created',
+            },
+        ),
+    ]
diff --git a/rootfs/api/migrations/0004_build_dryccfile_release_state_alter_service_ports.py b/rootfs/api/migrations/0004_build_dryccfile_release_state_alter_service_ports.py
new file mode 100644
index 000000000..17a8d029a
--- /dev/null
+++ b/rootfs/api/migrations/0004_build_dryccfile_release_state_alter_service_ports.py
@@ -0,0 +1,30 @@
+# Generated by Django 4.2.10 on 2024-04-03 02:03
+
+import api.models.service
+from django.db import migrations, models
+import functools
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0003_limitspec_remove_app_procfile_structure_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='dryccfile',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='state',
+            field=models.TextField(choices=[('created', 'Release created but not deployed'), ('crashed', 'Release pipeline runtime crashed'), ('succeed', 'Release pipeline runtime succeed')], default='created'),
+        ),
+        migrations.AlterField(
+            model_name='service',
+            name='ports',
+            field=models.JSONField(default=list, validators=[functools.partial(api.models.service.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'items': {'properties': {'name': {'type': 'string'}, 'port': {'type': 'integer'}, 'protocol': {'type': 'string'}, 'targetPort': {'type': 'integer'}}, 'required': ['name', 'port', 'protocol', 'targetPort'], 'type': 'object'}, 'minItems': 1, 'type': 'array'}})]),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0005_config_typed_values.py b/rootfs/api/migrations/0005_config_typed_values.py
new file mode 100644
index 000000000..40e5fb700
--- /dev/null
+++ b/rootfs/api/migrations/0005_config_typed_values.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-04-26 04:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0004_build_dryccfile_release_state_alter_service_ports'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='config',
+            name='typed_values',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0006_token.py b/rootfs/api/migrations/0006_token.py
new file mode 100644
index 000000000..5f3921504
--- /dev/null
+++ b/rootfs/api/migrations/0006_token.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.10 on 2024-04-30 02:02
+
+import api.utils
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import functools
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0005_config_typed_values'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Token',
+            fields=[
+                ('uuid', models.UUIDField(auto_created=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='UUID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('key', models.CharField(max_length=128, unique=True, verbose_name='Key')),
+                ('alias', models.CharField(blank=True, default='', max_length=32, verbose_name='Alias')),
+                ('oauth', models.JSONField(validators=[functools.partial(api.utils.validate_json, *(), **{'schema': {'$schema': 'http://json-schema.org/schema#', 'properties': {'access_token': {'type': 'string'}, 'expires_in': {'type': 'integer'}, 'refresh_token': {'type': 'string'}, 'scope': {'type': 'string'}, 'token_type': {'type': 'string'}}, 'required': ['access_token', 'expires_in', 'token_type', 'scope', 'refresh_token'], 'type': 'object'}})])),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]
diff --git a/rootfs/api/migrations/0007_domain_procfile_type.py b/rootfs/api/migrations/0007_domain_procfile_type.py
new file mode 100644
index 000000000..8727b5545
--- /dev/null
+++ b/rootfs/api/migrations/0007_domain_procfile_type.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.10 on 2024-05-01 13:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0006_token'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='domain',
+            name='procfile_type',
+            field=models.TextField(default='web'),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py b/rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py
new file mode 100644
index 000000000..6497eec76
--- /dev/null
+++ b/rootfs/api/migrations/0008_alter_domain_procfile_type_alter_route_procfile_type_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.10 on 2024-05-15 04:19
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0007_domain_procfile_type'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='domain',
+            name='procfile_type',
+            field=models.CharField(max_length=63),
+        ),
+        migrations.AlterField(
+            model_name='route',
+            name='procfile_type',
+            field=models.CharField(max_length=63),
+        ),
+        migrations.AlterField(
+            model_name='service',
+            name='procfile_type',
+            field=models.CharField(max_length=63),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py b/rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py
new file mode 100644
index 000000000..563a1e0d1
--- /dev/null
+++ b/rootfs/api/migrations/0009_remove_appsettings_canaries_remove_release_canary_and_more.py
@@ -0,0 +1,34 @@
+# Generated by Django 4.2.11 on 2024-07-01 07:58
+
+from django.db import migrations
+
+
+def migration_route_rules(apps, schema_editor):
+    route_model = apps.get_model('api', 'Route')
+    for route in route_model.objects.all():
+        if isinstance(route.rules, dict):
+            route.rules = route.rules.get("stable")
+        route.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0008_alter_domain_procfile_type_alter_route_procfile_type_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='appsettings',
+            name='canaries',
+        ),
+        migrations.RemoveField(
+            model_name='release',
+            name='canary',
+        ),
+        migrations.RemoveField(
+            model_name='service',
+            name='canary',
+        ),
+        migrations.RunPython(migration_route_rules),
+    ]
diff --git a/rootfs/api/migrations/0010_appsettings_autorollback.py b/rootfs/api/migrations/0010_appsettings_autorollback.py
new file mode 100644
index 000000000..aa4ab8f88
--- /dev/null
+++ b/rootfs/api/migrations/0010_appsettings_autorollback.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-08-13 07:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0009_remove_appsettings_canaries_remove_release_canary_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appsettings',
+            name='autorollback',
+            field=models.BooleanField(default=True),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0011_appsettings_autodeploy.py b/rootfs/api/migrations/0011_appsettings_autodeploy.py
new file mode 100644
index 000000000..5744324da
--- /dev/null
+++ b/rootfs/api/migrations/0011_appsettings_autodeploy.py
@@ -0,0 +1,27 @@
+# Generated by Django 4.2.15 on 2024-08-27 02:18
+
+from django.db import migrations, models
+
+
+def migration_dryccfile(apps, schema_editor):
+    build_model = apps.get_model('api', 'Build')
+    for build in build_model.objects.all():
+        if build.dryccfile and isinstance(build.dryccfile.get('run'), dict):
+            build.dryccfile['run'] = [build.dryccfile['run']]
+            build.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0010_appsettings_autorollback'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='appsettings',
+            name='autodeploy',
+            field=models.BooleanField(default=True),
+        ),
+        migrations.RunPython(migration_dryccfile),
+    ]
diff --git a/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py b/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
new file mode 100644
index 000000000..8214d715b
--- /dev/null
+++ b/rootfs/api/migrations/0012_alter_app_options_rename_procfile_type_domain_ptype_and_more.py
@@ -0,0 +1,57 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:57
+
+import api.utils
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_appsettings_autodeploy'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='app',
+            options={'ordering': ['id'], 'verbose_name': 'Application'},
+        ),
+        migrations.RenameField(
+            model_name='domain',
+            old_name='procfile_type',
+            new_name='ptype',
+        ),
+        migrations.RenameField(
+            model_name='route',
+            old_name='procfile_type',
+            new_name='ptype',
+        ),
+        migrations.RenameField(
+            model_name='service',
+            old_name='procfile_type',
+            new_name='ptype',
+        ),
+        migrations.AddField(
+            model_name='certificate',
+            name='app',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='api.app'),
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='conditions',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AlterField(
+            model_name='certificate',
+            name='name',
+            field=models.CharField(max_length=253, validators=[api.utils.validate_label]),
+        ),
+        migrations.AlterUniqueTogether(
+            name='certificate',
+            unique_together={('app', 'name')},
+        ),
+        migrations.AlterUniqueTogether(
+            name='service',
+            unique_together={('app', 'ptype')},
+        ),
+    ]
diff --git a/rootfs/api/migrations/0013_migration_permissions_and_certificates.py b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
new file mode 100644
index 000000000..63e87c4ab
--- /dev/null
+++ b/rootfs/api/migrations/0013_migration_permissions_and_certificates.py
@@ -0,0 +1,31 @@
+# Generated by Django 4.2.15 on 2024-09-03 03:48
+
+from django.db import migrations
+from api.models.domain import Domain
+from api.models.certificate import Certificate
+
+
+def migration_certificate(apps, schema_editor):
+    for domain in Domain.objects.all():
+        if domain.certificate:
+            certificate = Certificate.objects.filter(
+                app=domain.app, name=domain.certificate.name).first()
+            if not certificate:
+                certificate = domain.certificate
+                certificate.pk = None
+                certificate.app = domain.app
+                certificate.save()
+            domain.certificate = certificate
+            domain.save()
+    Certificate.objects.filter(app=None).delete()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0012_alter_app_options_rename_procfile_type_domain_ptype_and_more'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_certificate),
+    ]
diff --git a/rootfs/api/migrations/0014_alter_certificate_app.py b/rootfs/api/migrations/0014_alter_certificate_app.py
new file mode 100644
index 000000000..8d536fd7e
--- /dev/null
+++ b/rootfs/api/migrations/0014_alter_certificate_app.py
@@ -0,0 +1,19 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0013_migration_permissions_and_certificates'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='certificate',
+            name='app',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.app'),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py b/rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py
new file mode 100644
index 000000000..f3786e0a8
--- /dev/null
+++ b/rootfs/api/migrations/0015_alter_appsettings_autodeploy_and_more.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.15 on 2024-09-24 09:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0014_alter_certificate_app'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='appsettings',
+            name='autodeploy',
+            field=models.BooleanField(default=None),
+        ),
+        migrations.AlterField(
+            model_name='appsettings',
+            name='autorollback',
+            field=models.BooleanField(default=None),
+        ),
+        migrations.AlterField(
+            model_name='appsettings',
+            name='routable',
+            field=models.BooleanField(default=None),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py b/rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py
new file mode 100644
index 000000000..8f5f29df2
--- /dev/null
+++ b/rootfs/api/migrations/0016_remove_route_port_remove_route_ptype_and_more.py
@@ -0,0 +1,26 @@
+# Generated by Django 4.2.16 on 2024-10-21 04:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0015_alter_appsettings_autodeploy_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='route',
+            name='port',
+        ),
+        migrations.RemoveField(
+            model_name='route',
+            name='ptype',
+        ),
+        migrations.AddField(
+            model_name='release',
+            name='deployed_ptypes',
+            field=models.JSONField(default=list),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0017_config_values_refs.py b/rootfs/api/migrations/0017_config_values_refs.py
new file mode 100644
index 000000000..f1538341b
--- /dev/null
+++ b/rootfs/api/migrations/0017_config_values_refs.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-11-12 02:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0016_remove_route_port_remove_route_ptype_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='config',
+            name='values_refs',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0018_migration_config_values_and_typed_values.py b/rootfs/api/migrations/0018_migration_config_values_and_typed_values.py
new file mode 100644
index 000000000..b269d1844
--- /dev/null
+++ b/rootfs/api/migrations/0018_migration_config_values_and_typed_values.py
@@ -0,0 +1,43 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+import json
+from django.db import connection, migrations
+from api.models.config import Config
+
+
+def migration_values(apps, schema_editor):
+    with connection.cursor() as cursor:
+        cursor.execute("SELECT uuid,values,typed_values,registry FROM api_config")
+        for uuid, values, typed_values, registry in cursor:
+            new_values = []
+            config = Config.objects.get(pk=uuid)
+            for name, value in json.loads(values).items():
+                new_values.append({
+                    "name": name,
+                    "group": "global",
+                    "value": value
+                })
+            for ptype, values in json.loads(typed_values).items():
+                for name, value in values.items():
+                    new_values.append({
+                        "name": name,
+                        "ptype": ptype,
+                        "value": value
+                    })
+            new_registry, data = {}, json.loads(registry)
+            if data:
+                for ptype in config.app.structure.keys():
+                    new_registry[ptype] = data
+            config.values = new_values
+            config.registry = new_registry
+            config.save(ignore_update_fields=config.allof_fields)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0017_config_values_refs'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_values),
+    ]
diff --git a/rootfs/api/migrations/0019_remove_config_typed_values_alter_config_values.py b/rootfs/api/migrations/0019_remove_config_typed_values_alter_config_values.py
new file mode 100644
index 000000000..ee1a0b9fe
--- /dev/null
+++ b/rootfs/api/migrations/0019_remove_config_typed_values_alter_config_values.py
@@ -0,0 +1,22 @@
+# Generated by Django 4.2.16 on 2024-11-06 06:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0018_migration_config_values_and_typed_values'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='config',
+            name='typed_values',
+        ),
+        migrations.AlterField(
+            model_name='config',
+            name='values',
+            field=models.JSONField(blank=True, default=list),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0020_limitplan_runtime_class_name.py b/rootfs/api/migrations/0020_limitplan_runtime_class_name.py
new file mode 100644
index 000000000..c473af4e0
--- /dev/null
+++ b/rootfs/api/migrations/0020_limitplan_runtime_class_name.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.17 on 2024-12-25 08:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0019_remove_config_typed_values_alter_config_values'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='limitplan',
+            name='runtime_class_name',
+            field=models.CharField(default='', max_length=63),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0021_migration_dryccfile.py b/rootfs/api/migrations/0021_migration_dryccfile.py
new file mode 100644
index 000000000..3bd0140ba
--- /dev/null
+++ b/rootfs/api/migrations/0021_migration_dryccfile.py
@@ -0,0 +1,33 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+
+from django.db import migrations
+from api.models.release import Release
+
+
+def migration_dryccfile(apps, schema_editor):
+    for release in Release.objects.all():
+        build = release.build
+        if not build or not build.dryccfile or "pipeline" in build.dryccfile:
+            continue
+        pipeline = {}
+        for key, value in build.dryccfile.get('deploy', {}).items():
+            pipeline.update({
+                "%s.yaml" % key: {
+                    "kind": "pipeline",
+                    "ptype": key,
+                    "deploy": value,
+                }
+            })
+        build.dryccfile = {"pipeline": pipeline}
+        build.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0020_limitplan_runtime_class_name'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_dryccfile),
+    ]
diff --git a/rootfs/api/migrations/0022_migration_deployed_ptypes.py b/rootfs/api/migrations/0022_migration_deployed_ptypes.py
new file mode 100644
index 000000000..b5a9768ac
--- /dev/null
+++ b/rootfs/api/migrations/0022_migration_deployed_ptypes.py
@@ -0,0 +1,28 @@
+# Generated by Django 4.2.15 on 2024-09-06 03:58
+
+from django.db import migrations
+from api.models.release import Release
+
+
+def migration_deployed_ptypes(apps, schema_editor):
+    for release in Release.objects.all():
+        deployed_ptypes = set()
+        for condition in release.conditions:
+            if condition["ptypes"] is None:
+                deployed_ptypes = release.ptypes
+                break
+            else:
+                deployed_ptypes = deployed_ptypes.union(condition["ptypes"])
+        release.deployed_ptypes = list(deployed_ptypes)
+        release.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0021_migration_dryccfile'),
+    ]
+
+    operations = [
+        migrations.RunPython(migration_deployed_ptypes),
+    ]
\ No newline at end of file
diff --git a/rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py b/rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py
new file mode 100644
index 000000000..0adac79fa
--- /dev/null
+++ b/rootfs/api/migrations/0023_migration_k8s_alter_volume_type.py
@@ -0,0 +1,26 @@
+# Generated by Django 4.2.18 on 2025-03-26 02:44
+
+from django.db import migrations, models
+from api.models.volume import Volume
+
+
+def save_volume_to_k8s(apps, schema_editor):
+    for volume in Volume.objects.all():
+        if volume.type == 'nfs':
+            volume.save_to_k8s()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0022_migration_deployed_ptypes'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='volume',
+            name='type',
+            field=models.CharField(choices=[('csi', 'container storage interface'), ('nfs', 'network file system'), ('oss', 'object storage service file')], default='csi'),
+        ),
+        migrations.RunPython(save_volume_to_k8s),
+    ]
diff --git a/rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py b/rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py
new file mode 100644
index 000000000..d6e8058e2
--- /dev/null
+++ b/rootfs/api/migrations/0024_limitplan_container_volume_mounts_and_more.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.22 on 2025-07-17 06:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0023_migration_k8s_alter_volume_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='limitplan',
+            name='container_volume_mounts',
+            field=models.JSONField(default=list),
+        ),
+        migrations.AddField(
+            model_name='limitplan',
+            name='pod_volumes',
+            field=models.JSONField(default=list),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py b/rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py
new file mode 100644
index 000000000..eeb228016
--- /dev/null
+++ b/rootfs/api/migrations/0025_alter_app_id_alter_gateway_name.py
@@ -0,0 +1,25 @@
+# Generated by Django 4.2.22 on 2025-08-22 02:48
+
+import api.models.app
+import api.utils
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0024_limitplan_container_volume_mounts_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='app',
+            name='id',
+            field=models.SlugField(max_length=63, null=True, unique=True, validators=[api.models.app.validate_app_id]),
+        ),
+        migrations.AlterField(
+            model_name='gateway',
+            name='name',
+            field=models.CharField(db_index=True, max_length=63, validators=[api.utils.validate_label]),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0026_app_suspended_state.py b/rootfs/api/migrations/0026_app_suspended_state.py
new file mode 100644
index 000000000..a667b809f
--- /dev/null
+++ b/rootfs/api/migrations/0026_app_suspended_state.py
@@ -0,0 +1,18 @@
+# Generated by Django 4.2.22 on 2025-08-22 02:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0025_alter_app_id_alter_gateway_name'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='app',
+            name='suspended_state',
+            field=models.JSONField(blank=True, default=dict),
+        ),
+    ]
diff --git a/rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py b/rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py
new file mode 100644
index 000000000..27cdfd3ff
--- /dev/null
+++ b/rootfs/api/migrations/0027_rename_lifecycle_post_start_config_lifecycle_and_more.py
@@ -0,0 +1,22 @@
+# Generated by Django 5.2.8 on 2025-12-05 13:39
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0026_app_suspended_state'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='config',
+            old_name='lifecycle_post_start',
+            new_name='lifecycle',
+        ),
+        migrations.RemoveField(
+            model_name='config',
+            name='lifecycle_pre_stop',
+        ),
+    ]
diff --git a/rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py b/rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py
new file mode 100644
index 000000000..5dfb1ee61
--- /dev/null
+++ b/rootfs/api/migrations/0028_workspace_remove_app_owner_remove_appsettings_owner_and_more.py
@@ -0,0 +1,119 @@
+# Generated by Django 5.2.9 on 2026-03-23 05:42
+
+import api.models.workspace
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0027_rename_lifecycle_post_start_config_lifecycle_and_more'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Workspace',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.SlugField(max_length=150, unique=True, validators=[api.models.workspace.validate_workspace_name], verbose_name='workspace name')),
+                ('email', models.EmailField(max_length=254, verbose_name='email address')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='app',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='appsettings',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='build',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='certificate',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='config',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='domain',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='gateway',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='release',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='route',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='resource',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='service',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='tls',
+            name='owner',
+        ),
+        migrations.RemoveField(
+            model_name='volume',
+            name='owner',
+        ),
+        migrations.AlterField(
+            model_name='blocklist',
+            name='type',
+            field=models.PositiveIntegerField(choices=[(1, 'app'), (2, 'workspace')]),
+        ),
+        migrations.AddField(
+            model_name='app',
+            name='workspace',
+            field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='api.workspace'),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name='WorkspaceInvitation',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('email', models.EmailField(max_length=254, verbose_name='email address')),
+                ('token', models.CharField(max_length=128, unique=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('accepted', models.BooleanField(default=False, verbose_name='accepted')),
+                ('inviter', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.workspace')),
+            ],
+            options={
+                'unique_together': {('email', 'workspace')},
+            },
+        ),
+        migrations.CreateModel(
+            name='WorkspaceMember',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('role', models.CharField(choices=[('admin', 'Admin'), ('member', 'Member'), ('viewer', 'Viewer')], max_length=50)),
+                ('alerts', models.BooleanField(default=True)),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                ('workspace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.workspace')),
+            ],
+            options={
+                'unique_together': {('user', 'workspace')},
+            },
+        ),
+    ]
diff --git a/rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py b/rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py
new file mode 100644
index 000000000..454a81db8
--- /dev/null
+++ b/rootfs/api/migrations/0029_key_api_key_fingerp_4e77c5_idx_and_more.py
@@ -0,0 +1,25 @@
+# Generated by Django 5.2.9 on 2026-03-26 08:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0028_workspace_remove_app_owner_remove_appsettings_owner_and_more'),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name='key',
+            index=models.Index(fields=['fingerprint'], name='api_key_fingerp_4e77c5_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='limitplan',
+            index=models.Index(fields=['spec', 'cpu', 'memory'], name='api_limitpl_spec_id_f95ef2_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='release',
+            index=models.Index(fields=['app', 'failed', 'state'], name='api_release_app_id_9849f2_idx'),
+        ),
+    ]
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
index fe761a52f..3e7914035 100644
--- a/rootfs/api/models/app.py
+++ b/rootfs/api/models/app.py
@@ -1,9 +1,4 @@
-import backoff
 import base64
-import math
-from collections import OrderedDict
-from datetime import datetime
-from docker import auth as docker_auth
 import functools
 import json
 import logging
@@ -14,26 +9,32 @@
 import time
 import socket
 from contextlib import closing
-from itertools import groupby
 from urllib.parse import urljoin
+from collections import OrderedDict
+from datetime import datetime, timezone
 
+from docker import auth as docker_auth
 from django.conf import settings
 from django.db import models
+from django.db.models import F, Func, Value, JSONField
 from django.contrib.auth import get_user_model
-from rest_framework.exceptions import ValidationError, NotFound
+from rest_framework.exceptions import ValidationError
 
-from api.utils import get_session
+from api.tasks import send_app_log
+from api.utils import get_httpclient, dict_diff
 from api.exceptions import AlreadyExists, DryccException, ServiceUnavailable
-from api.utils import generate_app_name, apply_tasks, unit_to_bytes, unit_to_millicpu
+from api.utils import (
+    CacheLock, DeployLock, generate_app_name, apply_tasks, validate_reserved_names)
 from scheduler import KubeHTTPException, KubeException
 from .gateway import Gateway, Route
+from .limit import LimitPlan
 from .config import Config
 from .service import Service
 from .release import Release
 from .tls import TLS
 from .appsettings import AppSettings
 from .volume import Volume
-from .base import UuidAuditedModel
+from .base import UuidAuditedModel, PTYPE_WEB, PTYPE_RUN, DEFAULT_HTTP_PORT
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -44,44 +45,38 @@ def validate_app_id(value):
     """
     Check that the value follows the kubernetes name constraints
     """
-    match = re.match(r'[a-z]([a-z0-9-]*[a-z0-9])?(?<!-canary)$', value)
+    match = re.match(r'^[a-z]([a-z0-9-]{3,}[a-z0-9])$', value)
     if not match:
         raise ValidationError("App name must start with an alphabetic character, cannot end with a"
                               + " hyphen and can only contain a-z (lowercase), 0-9 and hyphens.")
+    validate_reserved_names(value)
 
 
 def validate_app_structure(value):
     """Error if the dict values aren't ints >= 0"""
     try:
-        if any(int(v) < 0 for v in value.values()):
-            raise ValueError("Must be greater than or equal to zero")
+        for k, v in value.items():
+            if int(v) < 0:
+                raise ValueError("Must be greater than or equal to zero")
+            validate_reserved_names(k)
     except ValueError as err:
         raise ValidationError(str(err))
 
 
-def validate_reserved_names(value):
-    """A value cannot use some reserved names."""
-    if value in settings.DRYCC_RESERVED_NAMES:
-        raise ValidationError('{} is a reserved name.'.format(value))
-
-
 class App(UuidAuditedModel):
     """
     Application used to service requests on behalf of end-users
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     id = models.SlugField(max_length=63, unique=True, null=True,
-                          validators=[validate_app_id,
-                                      validate_reserved_names])
+                          validators=[validate_app_id])
+    workspace = models.ForeignKey('Workspace', on_delete=models.CASCADE)
     structure = models.JSONField(
         default=dict, blank=True, validators=[validate_app_structure])
-    procfile_structure = models.JSONField(
-        default=dict, blank=True, validators=[validate_app_structure])
+    suspended_state = models.JSONField(default=dict, blank=True)
 
     class Meta:
         verbose_name = 'Application'
-        permissions = (('use_app', 'Can use app'),)
         ordering = ['id']
 
     def save(self, *args, **kwargs):
@@ -96,7 +91,7 @@ def save(self, *args, **kwargs):
             self.release_set.latest()
         except Release.DoesNotExist:
             try:
-                if self._scheduler.ns.get(self.id).status_code == 200:
+                if self.scheduler.ns.get(self.id).status_code == 200:
                     # Namespace already exists
                     err = "{} already exists as a namespace in this kuberenetes setup".format(self.id)  # noqa
                     self.log(err, logging.INFO)
@@ -111,9 +106,43 @@ def save(self, *args, **kwargs):
 
         return application
 
+    def lock(self):
+        return CacheLock(f"app:lock:{self.id}")
+
+    def _default_actor(self):
+        member = self.workspace.workspacemember_set.select_related('user').first()
+        if member:
+            return member.user
+        raise DryccException("workspace has no members")
+
     @property
-    def types(self):
-        return list(self.procfile_structure.keys())
+    def ptypes(self):
+        return list(self.structure.keys())
+
+    @property
+    def scheduler(self):
+        """
+        Override @Base.AuditedModel.scheduler;
+        since the app itself doesn't have an app object context,
+        directly reference using ID instead.
+        """
+        scheduler = super(App, self).scheduler
+        scheduler.metadata["annotations"]["drycc.cc/app_id"] = str(self.id)
+        scheduler.metadata["annotations"]["drycc.cc/workspace_id"] = str(self.workspace_id)
+        return scheduler
+
+    def check_ptypes(self, ptypes: set):
+        """
+        check available procfile types
+        """
+        if not ptypes:
+            ptypes = self.ptypes
+        else:
+            invalid_ptypes = ptypes.difference(self.ptypes)
+            if len(invalid_ptypes) != 0:
+                raise DryccException("process type {} is not included in procfile".
+                                     format(','.join(invalid_ptypes)))
+        return ptypes
 
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this application.
@@ -124,6 +153,7 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.id, message, level)
         logger.log(level, "[{}]: {}".format(self.id, message))
 
     def create(self, *args, **kwargs):  # noqa
@@ -131,81 +161,51 @@ def create(self, *args, **kwargs):  # noqa
         Create a application with an initial config, settings, release, domain
         and k8s resource if needed
         """
-        try:
-            cfg = self.config_set.latest()
-        except Config.DoesNotExist:
-            cfg = Config.objects.create(owner=self.owner, app=self)
-
+        cfg = self._set_default_config()
         # Only create if no release can be found
         try:
             self.release_set.latest()
         except Release.DoesNotExist:
             Release.objects.create(
-                version=1, owner=self.owner, app=self,
+                version=1, app=self,
                 config=cfg, build=None
             )
 
         # create required minimum resources in k8s for the application
-        namespace = limits_name = quota_name = self.id
+        namespace = self.id
+        self.log('creating Namespace {} and services'.format(namespace), level=logging.DEBUG)
+        # Create essential resources
         try:
-            self.log('creating Namespace {} and services'.format(namespace), level=logging.DEBUG)
-            # Create essential resources
-            try:
-                self._scheduler.ns.get(namespace)
-            except KubeException:
-                try:
-                    self._scheduler.ns.create(namespace)
-                except KubeException as e:
-                    raise ServiceUnavailable('Could not create the Namespace in Kubernetes') from e
-
-            if settings.KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC != '':
-                quota_spec = json.loads(settings.KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC)
-                self.log('creating Quota {} for namespace {}'.format(quota_name, namespace),
-                         level=logging.DEBUG)
-                try:
-                    self._scheduler.quota.get(namespace, quota_name)
-                except KubeException:
-                    self._scheduler.quota.create(namespace, quota_name, spec=quota_spec)
-            if settings.KUBERNETES_NAMESPACE_DEFAULT_LIMIT_RANGES_SPEC != '':
-                limits_spec = json.loads(settings.KUBERNETES_NAMESPACE_DEFAULT_LIMIT_RANGES_SPEC)
-                self.log('creating LimitRanges {} for namespace {}'.format(limits_name, namespace),
-                         level=logging.DEBUG)
-                try:
-                    self._scheduler.limits.get(namespace, limits_name)
-                except KubeException:
-                    self._scheduler.limits.create(namespace, limits_name, spec=limits_spec)
-        except KubeException as e:
-            # Blow it all away only if something horrible happens
+            self.scheduler.ns.get(namespace)
+        except KubeException:
             try:
-                self._scheduler.ns.delete(namespace)
+                self.scheduler.ns.create(namespace)
             except KubeException as e:
-                # Just feed into the item below
-                raise ServiceUnavailable('Could not delete the Namespace in Kubernetes') from e
-
-            raise ServiceUnavailable('Kubernetes resources could not be created') from e
+                raise ServiceUnavailable('Could not create the Namespace in Kubernetes') from e
         try:
             self.appsettings_set.latest()
         except AppSettings.DoesNotExist:
-            AppSettings.objects.create(owner=self.owner, app=self)
+            AppSettings.objects.create(
+                app=self, routable=True, autodeploy=True, autorollback=True)
         try:
             self.tls_set.latest()
         except TLS.DoesNotExist:
-            TLS.objects.create(owner=self.owner, app=self)
+            TLS.objects.create(app=self)
 
     def delete(self, *args, **kwargs):
         """Delete this application including all containers"""
         self.log("deleting environment")
         try:
             # check if namespace exists
-            self._scheduler.ns.get(self.id)
+            self.scheduler.ns.get(self.id)
 
             try:
-                self._scheduler.ns.delete(self.id)
+                self.scheduler.ns.delete(self.id)
 
                 # wait 30 seconds for termination
                 for _ in range(30):
                     try:
-                        self._scheduler.ns.get(self.id)
+                        self.scheduler.ns.get(self.id)
                     except KubeHTTPException as e:
                         # only break out on a 404
                         if e.response.status_code == 404:
@@ -216,8 +216,6 @@ def delete(self, *args, **kwargs):
         except KubeHTTPException:
             # it's fine if the namespace does not exist - delete app from the DB
             pass
-
-        self._clean_app_logs()
         return super(App, self).delete(*args, **kwargs)
 
     def restart(self, **kwargs):  # noqa
@@ -225,24 +223,18 @@ def restart(self, **kwargs):  # noqa
          Restart deployments with the kubectl rollout api
         """
         deployments = []
-        app_settings = self.appsettings_set.latest()
-        if 'type' in kwargs and kwargs['type'] in self.structure:
-            if self.structure[kwargs['type']] > 0:
-                if kwargs['type'] in app_settings.canaries:
-                    deployments.append(self._get_job_id(kwargs['type'], True))
-                deployments.append(self._get_job_id(kwargs['type'], False))
-        else:
-            for scale_type, count in self.structure.items():
-                if count > 0:
-                    if scale_type in app_settings.canaries:
-                        deployments.append(self._get_job_id(kwargs['type'], True))
-                    deployments.append(self._get_job_id(scale_type, False))
+        if self.structure[kwargs['type']] > 0:
+            deployments.append(self._get_deployment_name(kwargs['type']))
         try:
             tasks = [
-                functools.partial(
-                    self._scheduler.deployment.restart,
-                    self.id,
-                    deployment
+                (
+                    functools.partial(
+                        self.scheduler.deployment.restart,
+                        self.id,
+                        deployment
+                    ),
+                    lambda future: self.log(
+                        f'restart {kwargs['type']} callback: {future.result()}'),
                 ) for deployment in deployments
             ]
             apply_tasks(tasks)
@@ -250,276 +242,232 @@ def restart(self, **kwargs):  # noqa
             err = "warning, some pods failed to restart:\n{}".format(str(e))
             self.log(err, logging.WARNING)
 
-    def scale(self, user, structure):  # noqa
-        if self.release_set.filter(failed=False).latest().build is None:
-            raise DryccException('No build associated with this release')
-        release = self.release_set.filter(failed=False).latest()
+    def scale(self, user, structure):
+        err_msg = None
+        release = Release.latest(self)
+        if (PTYPE_RUN in structure or release is None or release.build is None):
+            if PTYPE_RUN in structure:
+                err_msg = 'Cannot set scale for reserved types, procfile type is: run'
+            else:
+                err_msg = 'No build associated with this release'
+            self.log(err_msg, logging.WARNING)
+            raise DryccException(err_msg)
         app_settings = self.appsettings_set.latest()
-        if release.canary:
-            self._scale(
-                user,
-                structure,
-                self.release_set.filter(failed=False, canary=False).latest(),
-                app_settings
-            )
         self._scale(user, structure, release, app_settings)
 
-    def deploy(self, release, force_deploy=False, rollback_on_failure=True):  # noqa
+    def pipeline(self, release, ptypes, force_deploy=False):
+        prefix = f"[pipeline] release {release.version_name}"
+        try:
+            if release is not None and release.build is not None:
+                if release.build.dryccfile:
+                    for run in release.get_runners(ptypes):
+                        self.log(f"{prefix} starts running pipeline.run {run['image']}")
+                        job_name = self.run(
+                            self._default_actor(), run['image'], command=run['command'],
+                            args=run['args'], timeout=run['timeout'], expires=run['timeout'],
+                            envs=self._build_env_vars(release, run['ptype']),
+                        )
+                        state, labels = 'initializing', {'job-name': job_name}
+                        for count, state in enumerate(self.scheduler.pod.watch(
+                                self.id, labels, settings.DRYCC_PILELINE_RUN_TIMEOUT)):
+                            self.log(f"{prefix} waiting for pipeline.run: {state} * {count}")
+                        if state != 'down':
+                            raise DryccException(f'pipeline run state error: {state}')
+                self.log(f"{prefix} starts running...")
+                rollback_on_failure = self.appsettings_set.latest().autorollback
+                if not rollback_on_failure:
+                    self.log(f"{prefix} deploy do not rollback on failure")
+                self.deploy(release, ptypes, force_deploy, rollback_on_failure)
+
+            release.state, release.failed = "succeed", False
+            ptypes = list(ptypes) if ptypes is not None else ptypes
+            release.add_condition(state="succeed", action="pipeline", ptypes=ptypes)
+        except Exception as e:
+            release.failed, release.state, msg = True, "crashed", f"{type(e).__name__}: {e}"
+            ptypes = list(ptypes) if ptypes is not None else ptypes
+            release.add_condition(
+                state="crashed", action="pipeline", ptypes=ptypes, exception=msg)
+            self.log(f"{prefix} pipeline runtime error, {msg}", logging.ERROR)
+        finally:
+            DeployLock(self.pk).release(ptypes)  # release all locks
+            release.save(update_fields=["state", "failed"])  # avoid overwriting other fields
+        self.log(f"{prefix} run completed...")
+
+    def deploy(self, release, ptypes=None, force_deploy=False, rollback_on_failure=True):
         """
         Deploy a new release to this application
 
         force_deploy can be used when a deployment is broken, such as for Rollback
         """
-        if release.build is None:
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release')
-
         # use create to make sure minimum resources are created
         self.create()
         # Previous release
         prev_release = release.previous()
-
-        # set processes structure to default if app is new.
-        if self.structure == {}:
-            self.structure = self._default_structure(release)
-            self.procfile_structure = self._default_structure(release)
-            self.save()
-        # reset canonical process types if build type has changed.
-        else:
-            # find the previous release's build type
-            if prev_release and prev_release.build:
-                if prev_release.build.type != release.build.type:
-                    structure = self.structure.copy()
-                    # zero out canonical pod counts
-                    for proctype in structure.keys():
-                        if proctype == "web":
-                            structure[proctype] = 0
-                    # update with the default process type.
-                    structure.update(self._default_structure(release))
-                    self.structure = structure
-                    # if procfile structure exists then we use it
-                    if release.build.procfile and \
-                            release.build.sha and not \
-                            release.build.dockerfile:
-                        self.procfile_structure = release.build.procfile
-                    self.save()
-
-        # always set the procfile structure for any new release
-        if release.build.procfile:
-            self.procfile_structure = release.build.procfile
-            self.save()
-
-        # always set default config
-        self._set_default_config()
+        self._merge_structure(release, prev_release)
         # deploy application to k8s. Also handles initial scaling
         app_settings = self.appsettings_set.latest()
         volumes = self.volume_set.all()
         deploys = {}
         for scale_type, replicas in self.structure.items():
-            scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
-            if not release.canary or scale_type in app_settings.canaries:
-                deploys[scale_type] = self._gather_app_settings(
-                    release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
-
-        # Sort deploys so routable comes first
-        deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
-        # Check if any proc type has a Deployment in progress
-        self._check_deployment_in_progress(deploys, release, force_deploy)
-
-        try:
-            # create the application config in k8s (secret in this case) for all deploy objects
-            self.set_application_config(release)
-
-            # gather all proc types to be deployed
-            tasks = [
-                functools.partial(
-                    self._scheduler.deploy,
-                    namespace=self.id,
-                    name=self._get_job_id(scale_type, release.canary),
-                    image=release.image,
-                    entrypoint=self._get_entrypoint(scale_type),
-                    command=self._get_command(scale_type),
-                    **kwargs
-                ) for scale_type, kwargs in deploys.items()
-            ]
-            try:
-                apply_tasks(tasks)
-            except KubeException as e:
-                # Don't rollback if the previous release doesn't have a build which means
-                # this is the first build and all the previous releases are just config changes.
-                if (prev_release.canary == release.canary and
-                        rollback_on_failure and prev_release.build is not None):
-                    err = 'There was a problem deploying {}. Rolling back to release {}.'.format(
-                        'v{}'.format(release.version), "v{}".format(prev_release.version))
-                    # This goes in the log before the rollback starts
-                    self.log(err, logging.ERROR)
-                    # revert all process types to old release
-                    self.deploy(prev_release, force_deploy=True, rollback_on_failure=False)
-                    # let it bubble up
-                    raise DryccException('{}\n{}'.format(err, str(e))) from e
-
-                # otherwise just re-raise
-                raise
-        except Exception as e:
-            # This gets shown to the end user
-            err = '(app::deploy): {}'.format(e)
-            self.log(err, logging.ERROR)
-            raise ServiceUnavailable(err) from e
-        for procfile_type, value in deploys.items():
-            if procfile_type == "web":  # http
-                target_port = int(value.get('envs', {}).get('PORT', 5000))
-                self._create_default_ingress(procfile_type, target_port)
-            service = self.service_set.filter(procfile_type=procfile_type).first()
-            if not service:
+            if ptypes is not None and scale_type not in ptypes:
                 continue
-            if prev_release and prev_release.build:
-                continue
-            if procfile_type == "web":
-                self._verify_http_health(service, **deploys[procfile_type])
-            else:
-                self._verify_tcp_health(service, **deploys[procfile_type])
+            scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
+            deploys[scale_type] = self._gather_app_settings(
+                release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
+        self._deploy(
+            deploys, ptypes, prev_release, release, force_deploy, rollback_on_failure)
         # cleanup old release objects from kubernetes
-        self.cleanup_old()
-        release.cleanup_old()
+        if app_settings.autodeploy:
+            self.clean(release=release)
+        release.clean(ptypes)
 
-    def mount(self, user, volume):
-        if self.release_set.filter(failed=False).latest().build is None:
+    def mount(self, user, volume, structure=None):
+        release = Release.latest(self)
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release')
-        release = self.release_set.filter(failed=False).latest()
         app_settings = self.appsettings_set.latest()
-        if release.canary:
-            self._mount(
-                user,
-                volume,
-                self.release_set.filter(failed=False, canary=False).latest(),
-                app_settings
-            )
-        self._mount(user, volume, release, app_settings)
-
-    def cleanup_old(self):
-        names, app_settings = [], self.appsettings_set.latest()
-        for scale_type in self.structure.keys():
-            if scale_type in app_settings.canaries:
-                names.append(self._get_job_id(scale_type, True))
-            names.append(self._get_job_id(scale_type, False))
+        self._mount(user, volume, app_settings, structure=structure)
+
+    def clean(self, release=None, ptypes=None):
+        release = release if release else self.release_set.latest()
+        # scale ptype down to 0, the next deploy will delete
+        pre_release = release.previous(None)
+        if pre_release is not None:
+            removed = {}
+            for ptype in pre_release.ptypes:
+                if ptype not in release.ptypes and self.structure.get(ptype, 0) > 0:
+                    if ptypes is None or ptype in ptypes:
+                        removed[ptype] = 0
+            self.scale(self._default_actor(), removed)
+        self._merge_structure(release, pre_release)
+        # clean k8s resources, ptype not in structure
         labels = {'heritage': 'drycc'}
-        deployments = self._scheduler.deployments.get(self.id, labels=labels).json()["items"]
-        if deployments is not None:
-            for deployment in deployments:
-                name = deployment['metadata']['name']
-                if name not in names:
-                    self._scheduler.deployments.delete(self.id, name, True)
-        self.log(f"cleanup old kubernetes deployments for {self.id}")
-
-    @backoff.on_exception(backoff.expo, ServiceUnavailable, max_tries=3)
-    def logs(self, log_lines=str(settings.LOG_LINES)):
-        """Return aggregated log data for this application."""
-        url = "http://{}:{}/logs/{}?log_lines={}".format(
-            settings.LOGGER_HOST, settings.LOGGER_PORT, self.id, log_lines)
-        try:
-            r = requests.get(url)
-        # Handle HTTP request errors
-        except requests.exceptions.RequestException as e:
-            msg = "Error accessing drycc-logger using url '{}': {}".format(url, e)
-            logger.error(msg)
-            raise ServiceUnavailable(msg) from e
-
-        # Handle logs empty or not found
-        if r.status_code == 204 or r.status_code == 404:
-            logger.info("GET {} returned a {} status code".format(url, r.status_code))
-            raise NotFound('Could not locate logs')
-
-        # Handle unanticipated status codes
-        if r.status_code != 200:
-            logger.error("Error accessing drycc-logger: GET {} returned a {} status code"
-                         .format(url, r.status_code))
-            raise ServiceUnavailable('Error accessing drycc-logger')
-
-        # cast content to string since it comes as bytes via the requests object
-        return str(r.content.decode('utf-8'))
-
-    def run(self, user, command, volumes=None):
+        if ptypes:
+            labels["type__in"] = ptypes
+        resource_apis = [self.scheduler.deployments, self.scheduler.secret]
+        for api in resource_apis:
+            resources = api.get(self.id, labels=labels).json()["items"]
+            if resources is not None:
+                for resource in resources:
+                    name = resource['metadata']['name']
+                    ptype = resource['metadata'].get("labels", {}).get("type")
+                    if (ptype and ptype not in self.structure):
+                        api.delete(self.id, name, True)
+        self.log(f"cleanup old kubernetes resources for {self.id}")
+
+    def run(self, user, image=None, command=None, args=None, volumes=None,
+            timeout=3600, expires=3600, **kwargs):
         def pod_name(size=5, chars=string.ascii_lowercase + string.digits):
             return ''.join(random.choice(chars) for _ in range(size))
 
         """Run a one-off command in an ephemeral app container."""
-        release = self.release_set.filter(failed=False).latest()
-        if release.build is None:
+        release = Release.latest(self)
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release to run this command')
 
         app_settings = self.appsettings_set.latest()
         volume_list = []
         if volumes:
-            volume_objs = Volume.objects.filter(app=release.app, name__in=volumes.keys())
-            for _ in volume_objs:
-                _.path["run"] = volumes.get(_.name, None)  # noqa
-                volume_list.append(_)
-        data = self._gather_app_settings(release, app_settings, process_type='run', replicas=1, volumes=volume_list)  # noqa
-
-        # create application config and build the pod manifest
-        self.set_application_config(release)
-
-        scale_type = 'run'
-        name = self._get_job_id(scale_type, release.canary) + '-' + pod_name()
+            for volume in Volume.objects.filter(app=self, name__in=volumes.keys()):
+                volume.path[PTYPE_RUN] = volumes.get(volume.name, None)
+                volume_list.append(volume)
+        else:
+            for volume in Volume.objects.filter(app=self):
+                if PTYPE_RUN in volume.path.keys():
+                    volume_list.append(volume)
+        data = self._gather_app_settings(
+            release, app_settings, ptype=PTYPE_RUN,
+            replicas=1, volumes=volume_list)
+        data['restart_policy'] = 'Never'
+        data['active_deadline_seconds'] = timeout
+        data['ttl_seconds_after_finished'] = expires
+        name = self._get_deployment_name(PTYPE_RUN) + '-' + pod_name()
         self.log("{} on {} runs '{}'".format(user.username, name, command))
-
+        kwargs.update(data)
         try:
-            exit_code, output = self._scheduler.run(
-                self.id,
-                name,
-                release.image,
-                self._get_entrypoint(scale_type),
-                [command],
-                **data
-            )
-
-            return exit_code, output
+            # create application config and build the pod manifest
+            self.set_application_config(release, PTYPE_RUN)
+            self.scheduler.job.create(self.id, name, image, command, args, **kwargs)
         except Exception as e:
-            err = '{} (run): {}'.format(name, e)
+            err = '{} ({}): {}'.format(name, PTYPE_RUN, e)
             raise ServiceUnavailable(err) from e
+        return name
+
+    def describe_pod(self, pod_name):
+        def get_command_and_args(pod, container_name):
+            command, args = [], []
+            for container in pod["spec"]["containers"]:
+                if container["name"] == container_name:
+                    args = container.get("args", [])
+                    command = container.get("command", [])
+                    break
+            return command, args
+        result = []
+        try:
+            pod = self.scheduler.pod.get(self.id, pod_name).json()
+            if pod["status"]['phase'] == 'Pending':
+                statuses = pod["spec"]["containers"]
+            else:
+                statuses = pod["status"]["containerStatuses"]
+            for status in statuses:
+                command, args = get_command_and_args(pod, status["name"])
+                result.append({
+                    "container": status["name"],
+                    "image": status["image"],
+                    "command": command,
+                    "args": args,
+                    "state": status.get("state", {}),
+                    "lastState": status.get("lastState", {}),
+                    "ready": status.get("ready", False),
+                    "restartCount": status.get("restartCount", 0),
+                    "status": pod["status"].get("phase", ""),
+                    "reason": pod["status"].get("reason", ""),
+                    "message": pod["status"].get("message", "")
+                })
+        except KubeHTTPException as e:
+            if e.response.status_code != 404:
+                raise e
+        return result
 
     def list_pods(self, *args, **kwargs):
         """Used to list basic information about pods running for a given application"""
         try:
             labels = self._scheduler_filter(**kwargs)
-
             # in case a singular pod is requested
             if 'name' in kwargs:
-                pods = [self._scheduler.pod.get(self.id, kwargs['name']).json()]
+                pods = [self.scheduler.pod.get(self.id, kwargs['name']).json()]
             else:
-                pods = self._scheduler.pod.get(self.id, labels=labels).json()['items']
+                pods = self.scheduler.pod.get(self.id, labels=labels).json()['items']
                 if not pods:
                     pods = []
-
             data = []
             for p in pods:
                 labels = p['metadata']['labels']
-                # specifically ignore run pods
-                if labels['type'] == 'run':
-                    continue
-
-                state = str(self._scheduler.pod.state(p))
-
-                # follows kubelete convention - these are hidden unless show-all is set
-                if state in ['down', 'crashed']:
-                    continue
-
-                # hide pod if it is passed the graceful termination period
-                if self._scheduler.pod.deleted(p):
-                    continue
-
-                item = {}
-                item['name'] = p['metadata']['name']
-                item['state'] = state
-                item['release'] = labels['version']
-                item['type'] = labels['type']
                 if 'startTime' in p['status']:
                     started = p['status']['startTime']
                 else:
-                    started = str(datetime.utcnow().strftime(settings.DRYCC_DATETIME_FORMAT))
-                item['started'] = started
-
+                    started = str(
+                        datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT))
+                state = str(self.scheduler.pod.state(p))
+                if p['status']['phase'] != 'Pending':
+                    ready = len([1 for s in p["status"]["containerStatuses"] if s['ready']])
+                    restarts = sum([s['restartCount'] for s in p["status"]["containerStatuses"]])
+                else:
+                    restarts = 0
+                    ready = 0
+                item = {
+                    'name': p['metadata']['name'],
+                    'state': state,
+                    'release': labels['version'], 'type': labels['type'], 'started': started,
+                    'ready': "%s/%s" % (
+                        ready,
+                        len(p["spec"]["containers"]),
+                    ),
+                    'restarts': restarts
+                }
                 data.append(item)
-
             # sorting so latest start date is first
             data.sort(key=lambda x: x['started'], reverse=True)
             return data
@@ -530,10 +478,116 @@ def list_pods(self, *args, **kwargs):
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
 
+    def delete_pod(self, **kwargs):
+        """Used to list basic information about pods running for a given application"""
+        pod_name = kwargs.get('pod_name')
+        try:
+            # make sure the pod is manageed by drycc
+            pod = self.scheduler.pod.get(self.id, pod_name).json()
+            if pod['metadata']['labels'].get("heritage") == "drycc":
+                self.scheduler.pod.delete(self.id, pod_name)
+        except KubeHTTPException as e:
+            # Sometimes k8s will manage to remove the pod from under us
+            if e.response.status_code != 404:
+                raise e
+
+    def describe_deployment(self, deployment_name):
+        result = []
+        try:
+            deployment = self.scheduler.deployment.get(self.id, deployment_name).json()
+            for container in deployment["spec"]["template"]['spec']["containers"]:
+                limits = container.get("resources", {}).get("limits", {})
+                result.append({
+                    "container": container["name"],
+                    "image": container["image"],
+                    "command": container.get("command", []),
+                    "args": container.get("args", []),
+                    "liveness_probe": container.get("livenessProbe", {}),
+                    "readiness_probe": container.get("readinessProbe", {}),
+                    "limits": limits,
+                    "volume_mounts": container.get("volumeMounts", []),
+                    "node_selector": deployment["spec"]["template"]['spec'].get("nodeSelector", {})  # noqa
+                })
+        except KubeHTTPException as e:
+            if e.response.status_code != 404:
+                raise e
+        return result
+
+    def list_deployments(self, *args, **kwargs):
+        """Used to list basic information about deployments running for a given application"""
+        ptypes = self.release_set.latest().ptypes
+        try:
+            labels = self._scheduler_filter(**kwargs)
+            # in case a singular deployment is requested
+            if 'name' in kwargs:
+                deployments = [self.scheduler.deployment.get(self.id, kwargs['name']).json()]
+            else:
+                deployments = self.scheduler.deployment.get(self.id, labels=labels).json()['items']  # noqa
+                if not deployments:
+                    deployments = []
+            data = []
+            for p in deployments:
+                labels = p['spec']['template']['metadata']['labels']
+                if p['metadata']['creationTimestamp']:
+                    started = p['metadata']['creationTimestamp']
+                else:
+                    started = str(
+                        datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT))
+                item = {
+                    'name': labels['type'],
+                    'release': labels['version'],
+                    'ready': "%s/%s" % (
+                        p["status"].get("readyReplicas", 0),
+                        p['spec'].get("replicas", 0),
+                    ),
+                    'garbage': False if labels['type'] in ptypes else True,
+                    'up_to_date': p["status"].get("updatedReplicas", 0),
+                    'available_replicas': p["status"].get("availableReplicas", 0),
+                    'started': started
+                }
+                data.append(item)
+            # sorting so latest start date is first
+            data.sort(key=lambda x: x['started'], reverse=True)
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = '(list deployments): {}'.format(e)
+            self.log(err, logging.ERROR)
+            raise ServiceUnavailable(err) from e
+
+    def list_events(self, ref_kind, ref_name, *args, **kwargs):
+        try:
+            fields = {
+                "regarding.kind": ref_kind,
+                "regarding.name": ref_name
+            }
+            kwargs["fields"] = fields
+            events = self.scheduler.events.get(self.id, **kwargs).json()['items']  # noqa
+            data = []
+            for e in events:
+                item = {
+                    'reason': e['reason'],
+                    'message': e['note'],
+                    'created': e['metadata']['creationTimestamp']
+                }
+                data.append(item)
+            # sorting so latest start date is first
+            data.sort(key=lambda x: x['created'], reverse=False)
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = '(list event): {}'.format(e)
+            self.log(err, logging.ERROR)
+            raise ServiceUnavailable(err) from e
+
     def autoscale(self, proc_type, autoscale):
         """
         Set autoscale rules for the application
         """
+        if proc_type == PTYPE_RUN:
+            raise DryccException('Cannot set autoscale for reserved types, procfile type is: run')
         name = '{}-{}'.format(self.id, proc_type)
         # basically fake out a Deployment object (only thing we use) to assign to the HPA
         target = {
@@ -543,43 +597,43 @@ def autoscale(self, proc_type, autoscale):
 
         try:
             # get the target for autoscaler, in this case Deployment
-            self._scheduler.hpa.get(self.id, name)
+            self.scheduler.hpa.get(self.id, name)
             if autoscale is None:
-                self._scheduler.hpa.delete(self.id, name)
+                self.scheduler.hpa.delete(self.id, name)
             else:
-                self._scheduler.hpa.update(
+                self.scheduler.hpa.update(
                     self.id, name, proc_type, target, **autoscale
                 )
         except KubeHTTPException as e:
             if e.response.status_code == 404:
-                self._scheduler.hpa.create(
+                self.scheduler.hpa.create(
                     self.id, name, proc_type, target, **autoscale
                 )
             else:
                 # let the user know about any other errors
                 raise ServiceUnavailable(str(e)) from e
 
-    def image_pull_secret(self, namespace, registry, image):
+    def image_pull_secret(self, namespace, ptype, registry, image):
         """
         Take registry information and set as an imagePullSecret for an RC / Deployment
         http://kubernetes.io/docs/user-guide/images/#specifying-imagepullsecrets-on-a-pod
         """
-        docker_config, name, create = self._get_private_registry_config(image, registry)
+        docker_config, name, create = self._get_private_registry_config(ptype, image, registry)
         if create is None:
             return
         elif create:
             data = {'.dockerconfigjson': docker_config}
             try:
-                self._scheduler.secret.get(namespace, name)
+                self.scheduler.secret.get(namespace, name)
             except KubeHTTPException:
-                self._scheduler.secret.create(
+                self.scheduler.secret.create(
                     namespace,
                     name,
                     data,
                     secret_type='kubernetes.io/dockerconfigjson'
                 )
             else:
-                self._scheduler.secret.update(
+                self.scheduler.secret.update(
                     namespace,
                     name,
                     data,
@@ -588,277 +642,306 @@ def image_pull_secret(self, namespace, registry, image):
 
         return name
 
-    def set_application_config(self, release):
+    def state_to_k8s(self):
+        release = Release.latest(self)
+        if release is None or release.build is None:
+            self.log('the last release does not have a build, skipping deployment...')
+            return
+        ptypes = set()
+        for ptype, scale in self.structure.items():
+            response = self.scheduler.deployment.get(
+                self.id, self._get_deployment_name(ptype),
+                ignore_exception=True)
+            if response.status_code == 404 and scale > 0:
+                ptypes.add(ptype)
+            elif response.status_code != 200:
+                data = response.json()
+                self.log('get deployment status_code {}, message: {}'.format(
+                    response.status_code, data.get("message", "")), logging.ERROR)
+        if len(ptypes) == 0:
+            self.log('the cluster status is the latest, skipping deployment...')
+            return
+        self.deploy(release, ptypes, False, False)
+
+    def set_application_config(self, release, ptype):
         """
         Creates the application config as a secret in Kubernetes and
         updates it if it already exists
         """
         # env vars are stored in secrets and mapped to env in k8s
-        version = 'v{}'.format(release.version)
         labels = {
-            'version': version,
-            'type': 'env'
+            'version': release.version_name,
+            'type': ptype,
+            'class': 'env'
         }
 
         # secrets use dns labels for keys, map those properly here
         secrets_env = {}
-        for key, value in self._build_env_vars(release).items():
+        for key, value in self._build_env_vars(release, ptype).items():
             secrets_env[key.lower().replace('_', '-')] = str(value)
 
         # dictionary sorted by key
         secrets_env = OrderedDict(sorted(secrets_env.items(), key=lambda t: t[0]))
 
-        secret_name = "{}-{}-env".format(self.id, version)
+        secret_name = "{}-{}-{}-env".format(self.id, ptype, release.version_name)
         try:
-            self._scheduler.secret.get(self.id, secret_name)
+            self.scheduler.secret.get(self.id, secret_name)
         except KubeHTTPException:
-            self._scheduler.secret.create(self.id, secret_name, secrets_env, labels=labels)
+            self.scheduler.secret.create(self.id, secret_name, secrets_env, labels=labels)
         else:
-            self._scheduler.secret.update(self.id, secret_name, secrets_env, labels=labels)
+            self.scheduler.secret.update(self.id, secret_name, secrets_env, labels=labels)
 
-    def to_measurements(self, timestamp: float):
-        measurements = []
+    def to_usages(self, timestamp: float):
+        usage = []
         config = self.config_set.latest()
-        for container_type, scale in self.structure.items():
-            measurements.append({
+        for ptype, scale in self.structure.items():
+            plan = config.limits.get(ptype)
+            usage.append({
                 "app_id": str(self.uuid),
-                "owner": self.owner_id,
-                "name": container_type,
-                "type": "cpu",
-                "unit": "milli",
-                "usage": unit_to_millicpu(config.cpu.get(container_type)) * scale,
-                "timestamp": int(timestamp)
+                "workspace": self.workspace_id,
+                "name": plan,
+                "type": "limits",
+                "unit": "number",
+                "usage": scale,
+                "kwargs": {
+                    "ptype": ptype,
+                },
+                "timestamp": int(timestamp),
+                "identifier": self.uuid.hex,
             })
-            measurements.append({
-                "app_id": str(self.uuid),
-                "owner": self.owner_id,
-                "name": container_type,
-                "type": "memory",
-                "unit": "bytes",
-                "usage": unit_to_bytes(config.memory.get(container_type)) * scale,
-                "timestamp": int(timestamp)
-            })
-        return measurements
+        return usage
 
     def __str__(self):
         return self.id
 
-    def _get_job_id(self, container_type, canary):
-        job_id = f"{self.id}-{container_type}"
-        if canary:
-            job_id = f"{job_id}-canary"
-        return job_id
+    def _get_deployment_name(self, ptype):
+        return f"{self.id}-{ptype}"
 
-    def _get_command(self, container_type):
-        """
-        Return the kubernetes "container arguments" to be sent off to the scheduler.
-        """
-        release = self.release_set.filter(failed=False).latest()
-        if release is not None and release.build is not None:
-            # dockerfile or container image
-            if release.build.dockerfile or not release.build.sha:
-                # has profile
-                if release.build.procfile and container_type in release.build.procfile:
-                    command = release.build.procfile[container_type]
-                    # if the entrypoint is `/bin/bash -c`, we want to supply the list
-                    # as a script. Otherwise, we want to send it as a list of arguments.
-                    if self._get_entrypoint(container_type) == ['/bin/sh', '-c']:
-                        return [command]
-                    else:
-                        return command.split()
-        return []
-
-    def _get_stack(self, release):
-        stack = release.config.values.get("DRYCC_STACK", None)
-        if stack is None:
-            if release.build.procfile \
-                    and release.build.sha \
-                    and not release.build.dockerfile:
-                stack = "buildpack"
-            else:
-                stack = "container"
-        return stack
-
-    def _get_entrypoint(self, container_type):
-        """
-        Return the kubernetes "container command" to be sent off to the scheduler.
-        """
-        entrypoint = []
-        release = self.release_set.filter(failed=False).latest()
-        if release is not None and release.build is not None:
-            if (release.build.procfile and container_type in release.build.procfile) \
-                    or container_type == "run":
-                entrypoint = ['/bin/sh', '-c']
-        if self._get_stack(release) == "buildpack":
-            if container_type in release.build.procfile:
-                entrypoint = [container_type]
-            else:
-                entrypoint = ['launcher']
-        return entrypoint
-
-    def _clean_app_logs(self):
-        """Delete application logs stored by the logger component"""
-        try:
-            url = 'http://{}:{}/logs/{}'.format(settings.LOGGER_HOST,
-                                                settings.LOGGER_PORT, self.id)
-            requests.delete(url)
-        except Exception as e:
-            # Ignore errors deleting application logs.  An error here should not interfere with
-            # the overall success of deleting an application, but we should log it.
-            err = 'Error deleting existing application logs: {}'.format(e)
-            self.log(err, logging.WARNING)
-
-    def _mount(self, user, volume, release, app_settings):
-        volumes = list(Volume.objects.filter(app=self).exclude(name=volume.name))
-        volumes.append(volume)
+    def _mount(self, user, volume, app_settings, structure=None):
+        volumes = Volume.objects.filter(app=self)
         tasks = []
-        for scale_type, replicas in self.structure.items():
-            if not release.canary or scale_type in app_settings.canaries:
+        for scale_type, replicas in structure.items() if structure else self.structure.items():
+            if scale_type != PTYPE_RUN:
+                release = self.release_set.filter(
+                    deployed_ptypes__contains=scale_type,
+                    failed=False).latest()
                 replicas = self.structure.get(scale_type, 0)
-                scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
+                scale_type_volumes = [
+                    volume for volume in volumes if scale_type in volume.path.keys()]
                 data = self._gather_app_settings(
                     release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
-                deployment = self._scheduler.deployment.get(
-                    self.id, self._get_job_id(scale_type, release.canary)).json()
+                deployment = self.scheduler.deployment.get(
+                    self.id, self._get_deployment_name(scale_type)).json()
                 spec_annotations = deployment['spec']['template']['metadata'].get(
                     'annotations', {})
+                self.set_application_config(release, scale_type)
                 # gather volume proc types to be deployed
-                tasks.append(
+                tasks.append((
                     functools.partial(
-                        self._scheduler.deployment.patch,
+                        self.scheduler.deployment.patch,
                         namespace=self.id,
-                        name=self._get_job_id(scale_type, release.canary),
-                        image=release.image,
-                        entrypoint=self._get_entrypoint(scale_type),
-                        command=self._get_command(scale_type),
+                        name=self._get_deployment_name(scale_type),
+                        image=release.get_deploy_image(scale_type),
+                        command=release.get_deploy_command(scale_type),
+                        args=release.get_deploy_args(scale_type),
                         spec_annotations=spec_annotations,
                         resource_version=deployment["metadata"]["resourceVersion"],
                         **data
-                    )
-                )
+                    ),
+                    lambda future: self.log(
+                        f'mount {volume} for {scale_type} callback: {future.result()}'),
+                ))
         try:
-            # create the application config in k8s (secret in this case) for all deploy objects
-            self.set_application_config(release)
             apply_tasks(tasks)
         except Exception as e:
             err = f'(changed volume mount for {volume}: {e}'
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
+        self.log(f'{user.username} changed volume mount for {volume}')
 
-        msg = f'{user.username} changed volume mount for {volume}'
-        self.log(msg)
+    def _deploy(self, deploys, ptypes, prev_release,
+                release, force_deploy, rollback_on_failure):
+        # Sort deploys so routable comes first
+        deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable')))
+        # Check if any proc type has a Deployment in progress
+        self._check_deployment_in_progress(deploys, force_deploy)
+        try:
+            tasks = []
+            lock = DeployLock(self.pk)
+            for scale_type, kwargs in deploys.items():
+                self.set_application_config(release, scale_type)
+                tasks.append((
+                    functools.partial(
+                        self.scheduler.deploy,
+                        namespace=self.id,
+                        name=self._get_deployment_name(scale_type),
+                        image=release.get_deploy_image(scale_type),
+                        command=release.get_deploy_command(scale_type),
+                        args=release.get_deploy_args(scale_type),
+                        **kwargs
+                    ),
+                    lambda future: self.log(
+                        f'deploy and unlock callback: {[
+                            future.result(), lock.release([scale_type])]}'),
+                ))
+            try:
+                apply_tasks(tasks)
+            except KubeException as e:
+                # Don't rollback if the previous release doesn't have a build which means
+                # this is the first build and all the previous releases are just config changes.
+                if rollback_on_failure and prev_release is not None and prev_release.build is not None:  # noqa
+                    err = 'There was a problem deploying {}. Rolling back to release {}.'.format(
+                        release.version_name, prev_release.version_name)
+                    # This goes in the log before the rollback starts
+                    self.log(err, logging.ERROR)
+                    # revert all process types to old release
+                    self.deploy(prev_release, ptypes, True, False)
+                    # let it bubble up
+                    raise DryccException('{}\n{}'.format(err, str(e))) from e
+                # otherwise just re-raise
+                raise
+        except Exception as e:
+            # This gets shown to the end user
+            err = '(app::deploy): {}'.format(e)
+            self.log(err, logging.ERROR)
+            raise ServiceUnavailable(err) from e
+        for ptype in deploys.keys():
+            if ptype == PTYPE_WEB:  # http
+                target_port = release.get_port(ptype)
+                self._create_default_ingress(target_port)
+            service = self.service_set.filter(ptype=ptype).first()
+            if not service:
+                continue
+            if prev_release and prev_release.build:
+                continue
+            if ptype == PTYPE_WEB:
+                self._verify_http_health(service, **deploys[ptype])
+            else:
+                self._verify_tcp_health(service, **deploys[ptype])
 
     def _scale(self, user, structure, release, app_settings):  # noqa
         """Scale containers up or down to match requested structure."""
         # use create to make sure minimum resources are created
         self.create()
-
         # Validate structure
         try:
             for target, count in structure.copy().items():
                 structure[target] = int(count)
             validate_app_structure(structure)
         except (TypeError, ValueError, ValidationError) as e:
-            raise DryccException('Invalid scaling format: {}'.format(e))
-
-        # test for available process types
-        available_process_types = release.build.procfile or {}
-        for container_type in structure:
-            if self._get_stack(release) == "container":
-                continue  # allow container types in case we don't have the image source
-            if container_type not in available_process_types:
-                raise NotFound(
-                    'Container type {} does not exist in application'.format(container_type))
-
-        # merge current structure and the new items together
-        old_structure = self.structure
-        new_structure = old_structure.copy()
-        new_structure.update(structure)
-
-        if new_structure != self.structure:
+            err_msg = 'Invalid scaling format: {}'.format(e)
+            self.log(err_msg)
+            raise DryccException(err_msg)
+
+        new_scale = dict_diff(structure, self.structure).get("changed", {})
+        old_scale = dict_diff(self.structure, structure).get("changed", {})
+
+        if new_scale:
             try:
-                self._scale_pods(structure, release, app_settings)
+                self._scale_pods(new_scale, release, app_settings)
             except ServiceUnavailable:
                 # scaling failed, go back to old scaling numbers
-                self._scale_pods(old_structure, release, app_settings)
+                self._scale_pods(old_scale, release, app_settings)
                 raise
             # save new structure to the database
-            self.structure = new_structure
-            self.procfile_structure = release.build.procfile
-            self.save()
+            App.objects.filter(id=self.id).update(
+                structure=Func(
+                    F("structure"),
+                    Value(new_scale, JSONField()),
+                    function="jsonb_concat",
+                )
+            )
             msg = '{} scaled pods '.format(user.username) + ' '.join(
                 "{}={}".format(k, v) for k, v in list(structure.items()))
             self.log(msg)
-
             return True
-
         return False
 
     def _scale_pods(self, scale_types, release, app_settings):
         volumes = Volume.objects.filter(app=self).exclude(path={})
         tasks = []
         for scale_type, replicas in scale_types.items():
-            scale_type_volumes = [_ for _ in volumes if scale_type in _.path.keys()]
-            data = self._gather_app_settings(release, app_settings, scale_type, replicas, volumes=scale_type_volumes)  # noqa
+            scale_type_volumes = [
+                volume for volume in volumes if scale_type in volume.path.keys()]
+            data = self._gather_app_settings(
+                release, app_settings, scale_type, replicas, volumes=scale_type_volumes)
+            # create the application config in k8s (secret in this case) for all deploy objects
+            self.set_application_config(release, scale_type)
             # gather all proc types to be deployed
-            tasks.append(
+            tasks.append((
                 functools.partial(
-                    self._scheduler.scale,
+                    self.scheduler.scale,
                     namespace=self.id,
-                    name=self._get_job_id(scale_type, release.canary),
-                    image=release.image,
-                    entrypoint=self._get_entrypoint(scale_type),
-                    command=self._get_command(scale_type),
+                    name=self._get_deployment_name(scale_type),
+                    image=release.get_deploy_image(scale_type),
+                    command=release.get_deploy_command(scale_type),
+                    args=release.get_deploy_args(scale_type),
                     **data
-                )
-            )
+                ),
+                lambda future: self.log(f'scale {scale_type} callback: {future.result()}'),
+            ))
         try:
-            # create the application config in k8s (secret in this case) for all deploy objects
-            self.set_application_config(release)
             apply_tasks(tasks)
         except Exception as e:
             err = '(scale): {}'.format(e)
             self.log(err, logging.ERROR)
             raise ServiceUnavailable(err) from e
 
+    def _set_default_limit(self, config, ptype):
+        if ptype not in config.limits:
+            plan = LimitPlan.get_default()
+            config.limits[ptype] = plan.id
+            config.save(update_fields=['limits'])
+        return config
+
     def _set_default_config(self):
-        default_cpu = "{}m".format(settings.KUBERNETES_LIMITS_MIN_CPU)
-        default_memory = "{}M".format(settings.KUBERNETES_LIMITS_MIN_MEMORY)
-        config = self.config_set.latest()
-        new_cpu, new_memory = {}, {}
-        for _type in self.types:
-            new_cpu[_type] = config.cpu.get(_type, default_cpu)
-            new_memory[_type] = config.memory.get(_type, default_memory)
-        config.cpu = new_cpu
-        config.memory = new_memory
-        config.save()
-
-    def _create_default_ingress(self, procfile_type, target_port):
-        port = 80
+        plan = LimitPlan.get_default()
+        limits = {PTYPE_WEB: plan.id, PTYPE_RUN: plan.id}
+        try:
+            config = self.config_set.latest()
+            limits[PTYPE_WEB] = config.limits.get(PTYPE_WEB, plan.id)
+            limits[PTYPE_RUN] = config.limits.get(PTYPE_RUN, plan.id)
+        except Config.DoesNotExist:
+            config = Config.objects.create(app=self, limits=limits)
+        for ptype in self.ptypes:
+            limits[ptype] = config.limits.get(ptype, plan.id)
+        if limits != config.limits:
+            config.limits = limits
+            config.save(update_fields=['limits'])
+        return config
+
+    def _create_default_ingress(self, target_port):
         # create default service
         try:
-            service = self.service_set.filter(procfile_type=procfile_type).latest()
+            service = self.service_set.filter(ptype=PTYPE_WEB).latest()
         except Service.DoesNotExist:
-            service = Service(owner=self.owner, app=self, procfile_type=procfile_type)
-            service.add_port(port, "TCP", target_port)
+            service = Service(app=self, ptype=PTYPE_WEB)
+            service.add_port(DEFAULT_HTTP_PORT, "TCP", target_port)
             service.save()
+        else:
+            if service.update_port(DEFAULT_HTTP_PORT, "TCP", target_port):
+                service.save()
         # create default gateway
         try:
             gateway = self.gateway_set.filter(name=self.id).latest()
+            if gateway.change_default_tls():
+                gateway.save()
         except Gateway.DoesNotExist:
-            gateway = Gateway(app=self, owner=self.owner, name=self.id)
-            added, msg = gateway.add(port, "HTTP")
+            gateway = Gateway(app=self, name=self.id)
+            added, msg = gateway.add(DEFAULT_HTTP_PORT, "HTTP")
             if not added:
                 raise DryccException(msg)
             gateway.save()
         # create default route
         try:
-            self.route_set.filter(name=self.id).latest()
+            route = self.route_set.filter(name=self.id).latest()
+            if route.change_default_tls():
+                route.save()
         except Route.DoesNotExist:
-            route = Route(app=self, owner=self.owner, kind="HTTPRoute", name=self.id,
-                          port=port, procfile_type=service.procfile_type)
-            route.rules = route.default_rules
-            attached, msg = route.attach(gateway.name, port)
+            route = Route(app=self, kind="HTTPRoute", name=self.id,
+                          rules=[{"backendRefs": [{"kind": "Service", "name": service.name,
+                                                   "port": DEFAULT_HTTP_PORT, "weight": 100}]}])
+            attached, msg = route.attach(gateway.name, DEFAULT_HTTP_PORT)
             if not attached:
                 raise DryccException(msg)
             route.save()
@@ -894,9 +977,10 @@ def _verify_http_health(self, service, **kwargs):
         response = None
         for _ in range(10):
             try:
-                # http://docs.python-requests.org/en/master/user/advanced/#timeouts
-                response = get_session().get(url, timeout=req_timeout)
-                failed = False
+                with get_httpclient() as session:
+                    # http://docs.python-requests.org/en/master/user/advanced/#timeouts
+                    response = session.get(url, timeout=req_timeout)
+                    failed = False
             except requests.exceptions.RequestException:
                 # In case of a failure where response object is not available
                 failed = True
@@ -939,13 +1023,13 @@ def _verify_tcp_health(self, service, **kwargs):
                 else:
                     time.sleep(3)
 
-    def _check_deployment_in_progress(self, deploys, release, force_deploy=False):
+    def _check_deployment_in_progress(self, deploys, force_deploy=False):
         if force_deploy:
             return
         for scale_type, kwargs in deploys.items():
-            name = self._get_job_id(scale_type, release.canary)
+            name = self._get_deployment_name(scale_type)
             # Is there an existing deployment in progress?
-            in_progress, deploy_okay = self._scheduler.deployment.in_progress(
+            in_progress, deploy_okay = self.scheduler.deployment.in_progress(
                 self.id, name, kwargs.get("deploy_timeout"), kwargs.get("deploy_batches"),
                 kwargs.get("replicas"), kwargs.get("tags")
             )
@@ -953,64 +1037,70 @@ def _check_deployment_in_progress(self, deploys, release, force_deploy=False):
             if in_progress and not deploy_okay:
                 raise AlreadyExists('Deployment for {} is already in progress'.format(name))
 
-    @staticmethod
-    def _default_structure(release):
+    def _merge_structure(self, release, prev_release):
         """Scale to default structure based on release type"""
-        if release.build.sha and not release.build.dockerfile and \
-                (release.build.procfile and 'web' not in release.build.procfile):
-            structure = {}
-        # default to heroku workflow
-        else:
-            structure = {'web': 1}
-        return structure
+        lock = self.lock()
+        try:
+            lock.acquire()
+            self.refresh_from_db()
+            default_structure = {}
+            for ptype in release.ptypes:
+                default_structure[ptype] = 1 if ptype == PTYPE_WEB else 0
+            if (self.structure != default_structure) or (
+                prev_release and prev_release.build and
+                prev_release.build.type != release.build.type
+            ):
+                for ptype, scale in self.structure.items():
+                    # clean old ptype
+                    if ptype not in release.ptypes and scale == 0:
+                        continue
+                    default_structure[ptype] = scale
+                self.structure = default_structure
+                self.save()
+        finally:
+            lock.release()
+        return self.structure
 
     def _scheduler_filter(self, **kwargs):
         labels = {'app': self.id, 'heritage': 'drycc'}
-
-        # always supply a version, either latest or a specific one
-        if 'release' not in kwargs or kwargs['release'] is None:
-            release = self.release_set.filter(failed=False).latest()
-        else:
-            release = self.release_set.get(version=kwargs['release'])
-
-        version = "v{}".format(release.version)
-        labels.update({'version': version})
-
         if 'type' in kwargs:
             labels.update({'type': kwargs['type']})
-
+        if 'version' in kwargs:
+            if isinstance(kwargs['version'], int):
+                version = "v{}".format(kwargs['version'])
+            else:
+                version = kwargs['version']
+            labels.update({'version': version})
         return labels
 
-    def _build_env_vars(self, release):
+    def _build_env_vars(self, release, ptype):
         """
         Build a dict of env vars, setting default vars based on app type
         and then combining with the user set ones
         """
-        if release.build is None:
+        if release is None or release.build is None:
             raise DryccException('No build associated with this release to run this command')
 
         # mix in default environment information drycc may require
         default_env = {
             'DRYCC_APP': self.id,
-            'WORKFLOW_RELEASE': 'v{}'.format(release.version),
+            'WORKFLOW_RELEASE': release.version_name,
             'WORKFLOW_RELEASE_SUMMARY': release.summary,
             'WORKFLOW_RELEASE_CREATED_AT': str(release.created.strftime(
                 settings.DRYCC_DATETIME_FORMAT))
         }
 
         default_env['SOURCE_VERSION'] = release.build.sha
-
+        # merge envs on top of default to make envs win
+        default_env.update(release.config.envs(ptype))
         # fetch application port and inject into ENV vars as needed
-        port = release.get_port()
+        port = release.get_port(ptype)
         if port:
             default_env['PORT'] = port
-
-        # merge envs on top of default to make envs win
-        default_env.update(release.config.values)
         return default_env
 
-    def _get_private_registry_config(self, image, registry=None):
-        name = settings.REGISTRY_SECRET_PREFIX
+    def _get_private_registry_config(self, ptype, image, registry=None):
+        name = settings.REGISTRY_SECRET_PREFIX + '-' + ptype
         if registry:
             # try to get the hostname information
             hostname = registry.get('hostname', None)
@@ -1023,13 +1113,13 @@ def _get_private_registry_config(self, image, registry=None):
             username = registry.get('username')
             password = registry.get('password')
         elif settings.REGISTRY_LOCATION == 'off-cluster':
-            secret = self._scheduler.secret.get(
-                settings.WORKFLOW_NAMESPACE, 'registry-secret').json()
-            username = secret['data']['username']
-            password = secret['data']['password']
-            hostname = secret['data']['hostname']
+            secret = self.scheduler.secret.get(
+                settings.WORKFLOW_NAMESPACE, 'controller-creds').json()
+            hostname = secret['data']['registry-host']
             if hostname == '':
                 hostname = docker_auth.INDEX_URL
+            username = secret['data']['registry-username']
+            password = secret['data']['registry-password']
             name = name + '-' + settings.REGISTRY_LOCATION
         else:
             return None, None, None
@@ -1046,121 +1136,92 @@ def _get_private_registry_config(self, image, registry=None):
         })
         return docker_config, name, True
 
-    @staticmethod
-    def _get_request_cpu(size):
-        cpu_request_ratio = settings.KUBERNETES_REQUEST_CPU_RATIO
-        if size.isdigit():
-            unit = 'm'
-            num = (int(size) * 1000) / cpu_request_ratio
-        else:
-            num, unit = (
-                ''.join(item[1]) for item in groupby(
-                    size, key=lambda x: x.isdigit()
-                )
-            )
-            if unit not in ["m", "M"]:
-                raise DryccException("Units are represented in the number or milli of CPUs")
-            else:
-                num = int(num) / cpu_request_ratio
-        return "{num}{unit}".format(num=math.ceil(num), unit=unit)
-
-    @staticmethod
-    def _get_request_memory(size):
-        memory_request_ratio = settings.KUBERNETES_REQUEST_MEMORY_RATIO
-        num, unit = (
-            ''.join(item[1]) for item in groupby(
-                size, key=lambda x: x.isdigit()
-            )
-        )
-        if unit in ['G', 'g']:
-            unit = 'M'
-            num = (int(num) * 1024) / memory_request_ratio
-        elif unit in ['M', 'm']:
-            num = int(num) / memory_request_ratio
-        else:
-            raise DryccException('Units are represented in Megabytes(M), or Gigabytes (G)')
-        return "{num}{unit}".format(num=math.ceil(num), unit=unit)
-
-    def _gather_app_settings(self, release, app_settings, process_type, replicas, volumes=None):
+    def _get_volumes_and_mounts(self, ptype, volumes):
+        k8s_volumes, k8s_volume_mounts = [], []
+        if volumes:
+            for volume in volumes:
+                k8s_volumes.append(
+                    {"name": volume.name, "persistentVolumeClaim": {"claimName": volume.name}})
+                k8s_volume_mounts.append(
+                    {"name": volume.name, "mountPath": volume.path.get(ptype)})
+        return k8s_volumes, k8s_volume_mounts
+
+    def _gather_app_settings(self, release, app_settings, ptype, replicas, volumes=None):
         """
         Gathers all required information needed in one easy place for passing into
         the Kubernetes client to deploy an application
 
         Any global setting that can also be set per app goes here
         """
-        envs = self._build_env_vars(release)
-        config = release.config
-        cpu, memory = {}, {}
-        for key, value in config.cpu.items():
-            cpu[key] = "%s/%s" % (self._get_request_cpu(value), value)
-        for key, value in config.memory.items():
-            memory[key] = "%s/%s" % (self._get_request_memory(value), value)
+
+        envs = self._build_env_vars(release, ptype)
+        # Obtain a limit plan that must exist, if raise error here, it must be a bug
+        config = self._set_default_limit(release.config, ptype)
+        limit_plan = LimitPlan.objects.get(id=config.limits.get(ptype))
+
         # see if the app config has deploy batch preference, otherwise use global
-        batches = int(config.values.get('DRYCC_DEPLOY_BATCHES', settings.DRYCC_DEPLOY_BATCHES))  # noqa
+        batches = int(envs.get('DRYCC_DEPLOY_BATCHES', settings.DRYCC_DEPLOY_BATCHES))
 
         # see if the app config has deploy timeout preference, otherwise use global
-        deploy_timeout = int(config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT))  # noqa
+        deploy_timeout = int(envs.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT))
 
         # configures how many ReplicaSets to keep beside the latest version
-        deployment_history = config.values.get('KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT',
-                                               settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT)  # noqa
+        deployment_history = envs.get(
+            'KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT',
+            settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT)
 
         # get application level pod termination grace period
-        pod_termination_grace_period_seconds = int(config.values.get(
-            'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS))  # noqa
+        termination_grace_period_seconds = int(config.termination_grace_period.get(
+            ptype,
+            envs.get(
+                'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS',
+                settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS
+            )
+        ))
 
         # set the image pull policy that is associated with the application container
-        image_pull_policy = config.values.get('IMAGE_PULL_POLICY', settings.IMAGE_PULL_POLICY)
+        image_pull_policy = envs.get('IMAGE_PULL_POLICY', settings.IMAGE_PULL_POLICY)
 
+        # set registry
+        registry = config.registry.get(ptype, {})
         # create image pull secret if needed
-        image_pull_secret_name = self.image_pull_secret(self.id, config.registry, release.image)
+        image_pull_secret_name = self.image_pull_secret(
+            self.id, ptype, registry, release.get_deploy_image(ptype))
 
         # only web is routable
         # https://www.drycc.cc/applications/managing-app-processes/#default-process-types
-        routable = True if process_type == 'web' and app_settings.routable else False
-
-        healthcheck = config.get_healthcheck().get(process_type, {})
-        if not healthcheck and process_type == 'web':
-            healthcheck = config.get_healthcheck().get('web', {})
-        volumes_info = [{
-            "name": _.name,
-            "claimName": _.name,
-        } for _ in volumes] if volumes else []
-        volumes_info.extend(json.loads(settings.KUBERNETES_POD_DEFAULT_VOLUMES))
-        volume_mounts_info = [{
-            "name": _.name,
-            "mount_path": _.path.get(process_type),
-        } for _ in volumes] if volumes else []
-        volume_mounts_info.extend(json.loads(settings.KUBERNETES_POD_DEFAULT_VOLUME_MOUNTS))
+        routable = True if (
+            ptype == PTYPE_WEB and app_settings.routable) else False
 
+        volumes, volume_mounts = self._get_volumes_and_mounts(ptype, volumes)
+        volumes.extend(limit_plan.pod_volumes)
+        volume_mounts.extend(limit_plan.container_volume_mounts)
         return {
-            'memory': memory,
-            'cpu': cpu,
-            'tags': config.tags,
+            'tags': config.tags.get(ptype, {}),
             'envs': envs,
-            'registry': config.registry,
+            'registry': registry,
             'replicas': replicas,
-            'version': 'v{}'.format(release.version),
-            'app_type': process_type,
-            'resources': json.loads(settings.KUBERNETES_POD_DEFAULT_RESOURCES),
+            'version': release.version_name,
+            'app_type': ptype,
+            'resources': {"limits": limit_plan.limits, "requests": limit_plan.requests},
             'build_type': release.build.type,
-            'annotations': json.loads(settings.KUBERNETES_POD_DEFAULT_ANNOTATIONS),
-            'healthcheck': healthcheck,
-            'runtime_class_name': settings.DRYCC_APP_RUNTIME_CLASS,
+            'annotations': limit_plan.annotations,
+            'lifecycle': config.lifecycle.get(ptype, {}),
+            'healthcheck': config.healthcheck.get(ptype, {}),
+            'runtime_class_name': limit_plan.runtime_class_name,
             'dns_policy': settings.DRYCC_APP_DNS_POLICY,
-            'lifecycle_post_start': config.lifecycle_post_start,
-            'lifecycle_pre_stop': config.lifecycle_pre_stop,
             'routable': routable,
             'deploy_batches': batches,
             'restart_policy': "Always",
             'deploy_timeout': deploy_timeout,
             'deployment_revision_history_limit': deployment_history,
             'release_summary': release.summary,
-            'pod_termination_grace_period_seconds': pod_termination_grace_period_seconds,
-            'pod_termination_grace_period_each': config.termination_grace_period,
+            'termination_grace_period_seconds': termination_grace_period_seconds,
             'image_pull_secret_name': image_pull_secret_name,
             'image_pull_policy': image_pull_policy,
-            'volumes': volumes_info,
-            'volume_mounts': volume_mounts_info,
-            'security_context': json.loads(settings.KUBERNETES_POD_DEFAULT_SECURITY_CONTEXT),
+            'volumes': volumes,
+            'volume_mounts': volume_mounts,
+            'node_selector': limit_plan.node_selector,
+            'pod_security_context': limit_plan.pod_security_context,
+            'container_security_context': limit_plan.container_security_context,
         }
diff --git a/rootfs/api/models/appsettings.py b/rootfs/api/models/appsettings.py
index f26e317e7..32c1da8f7 100644
--- a/rootfs/api/models/appsettings.py
+++ b/rootfs/api/models/appsettings.py
@@ -1,14 +1,18 @@
 import logging
+
+from functools import partial
 from django.db import models
 from django.db import transaction
 
 from rest_framework.exceptions import NotFound
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.utils import dict_diff
 from api.exceptions import DryccException, AlreadyExists, UnprocessableEntity
 from .base import UuidAuditedModel
 
 User = get_user_model()
+logger = logging.getLogger(__name__)
 
 
 class AppSettings(UuidAuditedModel):
@@ -16,10 +20,10 @@ class AppSettings(UuidAuditedModel):
     Instance of Application settings used by scheduler
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    canaries = models.JSONField(default=list)
-    routable = models.BooleanField(default=True)
+    routable = models.BooleanField(default=None)
+    autodeploy = models.BooleanField(default=None)
+    autorollback = models.BooleanField(default=None)
     autoscale = models.JSONField(default=dict, blank=True)
     label = models.JSONField(default=dict, blank=True)
 
@@ -35,6 +39,18 @@ def __init__(self, *args, **kwargs):
     def __str__(self):
         return "{}-{}".format(self.app.id, str(self.uuid)[:7])
 
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this application.
+
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
+
     def previous(self):
         """
         Return the previous Release to this one.
@@ -51,28 +67,18 @@ def previous(self):
             prev_app_settings = None
         return prev_app_settings
 
-    def _update_canaries(self, previous_settings):
-        old = getattr(previous_settings, 'canaries', [])
-        new = getattr(self, 'canaries', [])
-        data = old.copy()
-        if data and not new:
-            setattr(self, 'canaries', data)
-        elif data != new:
-            for procfile_type in new:
-                if procfile_type not in data:
-                    data.append(procfile_type)
-            setattr(self, 'canaries', data)
-            self.summary += [
-                "{} add canaries for process types {}".format(self.owner, ','.join(new))]
-
-    def _update_routable(self, previous_settings):
-        old = getattr(previous_settings, 'routable', None)
-        new = getattr(self, 'routable', None)
+    def _update_field(self, field, previous_settings):
+        old = getattr(previous_settings, field, None)
+        new = getattr(self, field, None)
         # if nothing changed copy the settings from previous
         if new is None and old is not None:
-            setattr(self, 'routable', old)
+            setattr(self, field, old)
+        elif new is None and isinstance(self._meta.get_field(field), models.BooleanField):
+            setattr(self, field, True)
         elif old != new:
-            self.summary += ["{} changed routablity from {} to {}".format(self.owner, old, new)]
+            self.summary += [
+                "{} changed {} from {} to {}".format(self.app.workspace.name, field, old, new)
+            ]
 
     def _update_autoscale(self, previous_settings):
         data = getattr(previous_settings, 'autoscale', {}).copy()
@@ -112,7 +118,7 @@ def _update_autoscale(self, previous_settings):
             deleted = 'deleted autoscale for process type ' + deleted if deleted else ''
             changes = ', '.join(i for i in (added, changed, deleted) if i)
             if changes:
-                self.summary += ["{} {}".format(self.owner, changes)]
+                self.summary += ["{} {}".format(self.app.workspace.name, changes)]
 
     def _update_label(self, previous_settings):
         data = getattr(previous_settings, 'label', {}).copy()
@@ -144,19 +150,24 @@ def _update_label(self, previous_settings):
             if changes:
                 if self.summary:
                     self.summary += ' and '
-                self.summary += ["{} {}".format(self.owner, changes)]
+                self.summary += ["{} {}".format(self.app.workspace.name, changes)]
 
-    def _update_fields(self, ignore_update_fields=None):
+    @transaction.atomic
+    def save(self, ignore_update_fields=None, *args, **kwargs):
         previous_settings = None
         try:
             previous_settings = self.app.appsettings_set.latest()
         except AppSettings.DoesNotExist:
             pass
-        update_fields = ["canaries", "routable", "autoscale", "label"]
+        update_fields = ["routable", "autodeploy", "autorollback", "autoscale", "label"]
         try:
-            for update_field in update_fields:
-                if ignore_update_fields is None or update_field not in ignore_update_fields:
-                    getattr(self, "_update_%s" % update_field)(previous_settings)
+            for field in update_fields:
+                if ignore_update_fields is None or field not in ignore_update_fields:
+                    getattr(
+                        self,
+                        "_update_%s" % field,
+                        partial(self._update_field, field)
+                    )(previous_settings)
         except (UnprocessableEntity, NotFound):
             raise
         except Exception as e:
@@ -165,28 +176,7 @@ def _update_fields(self, ignore_update_fields=None):
 
         if not self.summary and previous_settings:
             self.delete()
-            raise AlreadyExists("{} changed nothing".format(self.owner))
-        summary = ' '.join(self.summary)
-        self.app.log('summary of app setting changes: {}'.format(summary), logging.DEBUG)
-
-    def diff_canaries(self):
-        prev_app_settings = self.previous()
-        action, canaries = None, []
-        if prev_app_settings is not None:
-            if prev_app_settings.canaries != self.canaries:
-                for procfile_type in self.canaries:  # add canary
-                    if procfile_type not in prev_app_settings.canaries:
-                        if action is None:
-                            action = "append"
-                        canaries.append(procfile_type)
-                for procfile_type in prev_app_settings.canaries:  # delete canary
-                    if procfile_type not in self.canaries:
-                        if action is None:
-                            action = "remove"
-                        canaries.append(procfile_type)
-        return prev_app_settings, action, canaries
-
-    @transaction.atomic
-    def save(self, ignore_update_field=None, *args, **kwargs):
-        self._update_fields(ignore_update_field)
+            raise AlreadyExists("{} changed nothing".format(self.app.workspace.name))
         super(AppSettings, self).save(**kwargs)
+        summary = ' '.join(self.summary)
+        self.log('summary of app setting changes: {}'.format(summary), logging.DEBUG)
diff --git a/rootfs/api/models/base.py b/rootfs/api/models/base.py
index feef9c977..f3310d3dc 100644
--- a/rootfs/api/models/base.py
+++ b/rootfs/api/models/base.py
@@ -1,12 +1,37 @@
 import uuid
-import importlib
+import time
+import string
+import random
+from datetime import timedelta
+from functools import partial
 from django.db import models
-from django.conf import settings
+from django.utils import timezone
 from django.contrib.auth.models import AbstractUser
 from django.utils.translation import gettext_lazy as _
 
 
-def get_anonymous_user_instance(user): return user(id=-1, username=settings.ANONYMOUS_USER_NAME)
+from api.utils import get_httpclient, validate_json, get_scheduler
+
+token_manager_oauth_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "access_token": {"type": "string"},
+        "expires_in": {"type": "integer"},
+        "token_type": {"type": "string"},
+        "scope": {"type": "string"},
+        "refresh_token": {"type": "string"},
+    },
+    "required": [
+        "access_token", "expires_in", "token_type", "scope", "refresh_token"
+    ],
+}
+PTYPE_WEB = "web"
+PTYPE_RUN = "run"
+DEFAULT_HTTP_PORT = 80
+DEFAULT_HTTPS_PORT = 443
+PTYPE_MIN_LENGTH = 3
+PTYPE_MAX_LENGTH = 63
 
 
 class AuditedModel(models.Model):
@@ -19,11 +44,27 @@ class Meta:
         """Mark :class:`AuditedModel` as abstract."""
         abstract = True
 
-    @classmethod
+    def app_log(self, app_id, msg):
+        with get_httpclient() as session:
+            session.post(
+                "",
+                json={
+                    "timestamp": int(time.time()),
+                    "log": msg,
+                    "kubernetes": {
+                        "namespace": app_id,
+                        "container_name": "drycc-controller",
+                    }
+                }
+            )
+
     @property
-    def _scheduler(cls):
-        mod = importlib.import_module(settings.SCHEDULER_MODULE)
-        return mod.SchedulerClient(settings.SCHEDULER_URL, settings.K8S_API_VERIFY_TLS)
+    def scheduler(self):
+        annotations = {}
+        if hasattr(self, 'app'):
+            annotations["drycc.cc/app_id"] = str(self.app.id)
+            annotations["drycc.cc/workspace_id"] = str(self.app.workspace_id)
+        return get_scheduler(metadata={"annotations": annotations})
 
 
 class UuidAuditedModel(AuditedModel):
@@ -44,3 +85,37 @@ class Meta:
 class User(AbstractUser):
     id = models.BigIntegerField(_('id'), primary_key=True)
     email = models.EmailField(_('email address'), unique=True)
+
+
+class Token(UuidAuditedModel):
+    key = models.CharField(_("Key"), max_length=128, unique=True)
+    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    alias = models.CharField(_("Alias"), max_length=32, blank=True, default="")
+    oauth = models.JSONField(
+        validators=[partial(validate_json, schema=token_manager_oauth_schema)])
+
+    @property
+    def fuzzy_key(self):
+        return f'{self.key[:14]}...{self.key[-14:]}'
+
+    def save(self, *args, **kwargs):
+        if not self.key:
+            self.key = self.generate_key()
+        return super().save(*args, **kwargs)
+
+    def expires(self):
+        if self.updated + timedelta(seconds=self.oauth['expires_in']) < timezone.now():
+            return True
+        return False
+
+    @classmethod
+    def generate_key(cls):
+        return ''.join(
+            [i if random.randint(0, 9) % 2 == 0 else i.upper() for i in uuid.uuid4().hex]
+        ) + ''.join(random.choices(string.ascii_letters, k=96))
+
+    def refresh_token(self):
+        from api.apps_extra.social_core.backends import DryccOIDC
+        drycc_open_connect = DryccOIDC()
+        self.oauth = drycc_open_connect.refresh_token(self.oauth['refresh_token'])
+        self.save()
diff --git a/rootfs/api/models/blocklist.py b/rootfs/api/models/blocklist.py
index 5f5f6eac3..e51a2b613 100644
--- a/rootfs/api/models/blocklist.py
+++ b/rootfs/api/models/blocklist.py
@@ -1,18 +1,22 @@
+import logging
+import functools
 from django.db import models
-from django.contrib.auth import get_user_model
+from scheduler import KubeHTTPException
+from api.exceptions import ServiceUnavailable
+from api.utils import apply_tasks
 from .app import App
 from .base import UuidAuditedModel
 
-User = get_user_model()
+
+logger = logging.getLogger(__name__)
 
 
 class Blocklist(UuidAuditedModel):
     """
-    You can block apps or users.
-    If a user is blocked, all apps owned by the user will be stopped.
-    The apps managed by the user will not be affected.
+    You can block apps or workspaces.
+    If a workspace is blocked, all apps under that workspace will be stopped.
     """
-    type_choices = [(1, "app", ), (2, "user")]
+    type_choices = [(1, "app", ), (2, "workspace")]
     id = models.CharField(max_length=128, db_index=True)
     type = models.PositiveIntegerField(choices=type_choices)
     remark = models.TextField(blank=True, null=True, default="Blocked for unknown reason")
@@ -20,8 +24,7 @@ class Blocklist(UuidAuditedModel):
     @property
     def related_apps(self):
         if self.type == 2:
-            user = User.objects.get(id=self.id)
-            return App.objects.filter(owner=user)
+            return App.objects.filter(workspace_id=self.id)
         else:
             return App.objects.filter(id=self.id)
 
@@ -35,9 +38,233 @@ def get_type(cls, name: str):
     @classmethod
     def get_blocklist(cls, app: App):
         return cls.objects.filter(
-            models.Q(id=app.id, type=1) | models.Q(id=app.owner_id, type=2)
+            models.Q(id=app.id, type=1) | models.Q(id=app.workspace_id, type=2)
         ).first()
 
     class Meta:
         ordering = ['-created']
         unique_together = (("id", "type"),)
+
+    def related_resource_deployments(self, app: App):
+        "get resource deployments"
+        try:
+            deployments = self.scheduler.deployment.get(app.id, labels={}).json()['items']  # noqa
+            if not deployments:
+                deployments = []
+            data = []
+            for d in deployments:
+                item = {
+                    'name': d['metadata']['name'],
+                    'replicas': d['spec'].get("replicas", 0),
+                }
+                data.append(item)
+            data.sort(key=lambda x: x['name'])
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(list resource deployments): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def related_resource_statefulsets(self, app: App):
+        "get resource statefulsets"
+        try:
+            statefulsets = self.scheduler.statefulset.get(app.id, labels={}).json()['items']  # noqa
+            if not statefulsets:
+                statefulsets = []
+            data = []
+            for s in statefulsets:
+                item = {
+                    'name': s['metadata']['name'],
+                    'replicas': s['spec'].get("replicas", 0),
+                }
+                data.append(item)
+            data.sort(key=lambda x: x['name'])
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(list resource statefulsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def related_resource_daemonsets(self, app: App):
+        "get resource daemonsets"
+        try:
+            daemonsets = self.scheduler.daemonset.get(app.id, labels={}).json()['items']  # noqa
+            if not daemonsets:
+                daemonsets = []
+            data = []
+            for d in daemonsets:
+                item = {
+                    'name': d['metadata']['name'],
+                    'affinity': d['spec']['template']['spec'].get('affinity', {}),
+                }
+                data.append(item)
+            data.sort(key=lambda x: x['name'])
+            return data
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(list resource daemonsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def suspended_state(self, app: App):
+        """
+        Get deployments/statefulsets/daemonsets with labels app.kubernetes.io/managed-by=Helm
+        Store in the suspended_state field of the app model
+        Format:
+        {"deployments": [{"name":"sample1", "replicas": 1}],
+        "statefulsets": [{"name":"sample1", "replicas": 1}],
+        "daemonsets": [{"name":"sample1", "affinity": {}}]}
+        """
+        suspended_state = {}
+        suspended_state['deployments'] = self.related_resource_deployments(app)
+        suspended_state['statefulsets'] = self.related_resource_statefulsets(app)
+        suspended_state['daemonsets'] = self.related_resource_daemonsets(app)
+        return suspended_state
+
+    def scale_resource_deployments(self, app: App, deployment_name: str, replicas=0):
+        """scale deployments"""
+        try:
+            deployment = self.scheduler.deployment.get(app.id, deployment_name).json()
+            self.scheduler.scales.update(app.id, deployment_name, replicas, deployment)
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(scale resource deployments): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def scale_resource_statefulsets(self, app: App, statefulset_name: str, replicas=0):
+        """scale statefulsets"""
+        try:
+            self.scheduler.statefulset.get(app.id, statefulset_name).json()
+            manifest = {
+                'spec': {
+                    'persistentVolumeClaimRetentionPolicy': {
+                        'whenScaled': 'Retain'
+                    },
+                    'replicas': replicas
+                }
+            }
+            self.scheduler.statefulset.patch(app.id, statefulset_name, manifest)
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(scale resource statefulsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def scale_resource_daemonsets(self, app: App, daemonset_name: str, manifest: dict):
+        """set affinity"""
+        try:
+            self.scheduler.daemonset.get(app.id, daemonset_name).json()
+            self.scheduler.daemonset.patch(app.id, daemonset_name, manifest)
+        except KubeHTTPException:
+            pass
+        except Exception as e:
+            err = f'(scale resource daemonsets): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
+
+    def scale_resources(self, app: App, suspended_state: dict, scale_type="block"):
+        "scale resources tasks"
+        tasks = []
+        deployments = suspended_state.get('deployments', [])
+        if not deployments:
+            deployments = []
+        for d in deployments:
+            if scale_type == "unblock":
+                replicas = d['replicas']
+            else:
+                replicas = 0
+            tasks.append((
+                functools.partial(
+                    self.scale_resource_deployments,
+                    app=app,
+                    deployment_name=d['name'],
+                    replicas=replicas
+                ),
+                lambda future, name=d["name"]: app.log(
+                    f'{scale_type} scale deployment {name} callback: {future.result()}',
+                )
+            ))
+
+        statefulsets = suspended_state.get('statefulsets', [])
+        if not statefulsets:
+            statefulsets = []
+        for s in statefulsets:
+            if scale_type == "unblock":
+                replicas = s['replicas']
+            else:
+                replicas = 0
+            tasks.append((
+                functools.partial(
+                    self.scale_resource_statefulsets,
+                    app=app,
+                    statefulset_name=s['name'],
+                    replicas=replicas
+                ),
+                lambda future, name=s["name"]: app.log(
+                    f'{scale_type} scale statefulset {name} callback: {future.result()}',
+                )
+            ))
+
+        daemonsets = suspended_state.get('daemonsets', [])
+        if not daemonsets:
+            daemonsets = []
+        for d in daemonsets:
+            if scale_type == "unblock":
+                manifest = {
+                    "spec": {
+                        "template": {
+                            "spec": {
+                                "affinity": d['affinity']
+                            }
+                        }
+                    }
+                }
+            else:
+                manifest = {
+                    "spec": {
+                        "template": {
+                            "spec": {
+                                "affinity": {
+                                    "nodeAffinity": {
+                                        "requiredDuringSchedulingIgnoredDuringExecution": {
+                                            "nodeSelectorTerms": [{
+                                                "matchExpressions": [{
+                                                    "key": "kubernetes.io/hostname",
+                                                    "operator": "In",
+                                                    "values": [
+                                                        "nohostname"
+                                                    ]
+                                                }]
+                                            }]
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            tasks.append((
+                functools.partial(
+                    self.scale_resource_daemonsets,
+                    app=app,
+                    daemonset_name=d['name'],
+                    manifest=manifest
+                ),
+                lambda future, name=d["name"]: app.log(
+                    f'{scale_type} scale daemonset {name} callback: {future.result()}',
+                )
+            ))
+        try:
+            apply_tasks(tasks)
+        except Exception as e:
+            err = f'({scale_type} scale resources): {e}'
+            logger.info(err)
+            raise ServiceUnavailable(err) from e
diff --git a/rootfs/api/models/build.py b/rootfs/api/models/build.py
index 277540a1e..96bee571b 100644
--- a/rootfs/api/models/build.py
+++ b/rootfs/api/models/build.py
@@ -1,9 +1,12 @@
 import logging
-from django.conf import settings
+from copy import deepcopy
+from collections import defaultdict
 from django.db import models
 from django.contrib.auth import get_user_model
-from api.exceptions import DryccException, Conflict
-from .base import UuidAuditedModel
+from api.exceptions import DryccException
+from .base import UuidAuditedModel, PTYPE_WEB
+from .config import Config
+from .release import Release
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -14,7 +17,6 @@ class Build(UuidAuditedModel):
     Instance of a software build used by runtime nodes
     """
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     image = models.TextField()
     stack = models.CharField(max_length=32)
@@ -22,6 +24,7 @@ class Build(UuidAuditedModel):
     # optional fields populated by builder
     sha = models.CharField(max_length=40, blank=True)
     procfile = models.JSONField(default=dict, blank=True)
+    dryccfile = models.JSONField(default=dict, blank=True)
     dockerfile = models.TextField(blank=True)
 
     class Meta:
@@ -40,6 +43,14 @@ def type(self):
             # container image (or any sort of image) used via drycc pull
             return 'image'
 
+    @property
+    def ptypes(self):
+        if self.dryccfile:
+            return list([pipeline['ptype'] for pipeline in self.dryccfile['pipeline'].values()])
+        if self.procfile:
+            return list(self.procfile.keys())
+        return [PTYPE_WEB]
+
     @property
     def source_based(self):
         """
@@ -53,86 +64,98 @@ def source_based(self):
     def version(self):
         return 'git-{}'.format(self.sha) if self.source_based else 'latest'
 
-    def create(self, user, *args, **kwargs):
-        app_settings = self.app.appsettings_set.latest()
-        latest_release = self.app.release_set.filter(failed=False).latest()
-        latest_version = self.app.release_set.latest().version
+    def merge(self, config):
+        if self.dryccfile and any({"values", "values_refs"} & set(config.diff(None).keys())):
+            dryccfile = deepcopy(self.dryccfile)
+            ptype_env, group_env = defaultdict(dict), defaultdict(dict)
+            for value in config.values:
+                if "ptype" in value:
+                    ptype_env[value["ptype"]][value["name"]] = value["value"]
+                if "group" in value:
+                    group_env[value["group"]][value["name"]] = value["value"]
+            dryccfile['config'] = dict(group_env)
+            for pipeline in dryccfile['pipeline'].values():
+                pipeline['env'] = ptype_env[pipeline['ptype']]
+                pipeline['config'] = config.values_refs.get(pipeline['ptype'], [])
+            return Build.objects.create(
+                app=self.app, image=self.image, stack=self.stack, sha=self.sha,
+                procfile=self.procfile, dryccfile=dryccfile, dockerfile=self.dockerfile,
+            )
+        return self
+
+    def get_image(self, ptype, default_image=None):
+        pipeline = self.get_pipeline(ptype)
+        if pipeline and 'build' in pipeline:
+            return self.image if ptype == 'web' else f'{self.image}-{ptype}'
+        return default_image if default_image else self.image
+
+    def get_pipeline(self, ptype):
+        pipelines = self.get_pipelines([ptype])
+        if len(pipelines) > 0:
+            return pipelines[0]
+        return {}
+
+    def get_pipelines(self, ptypes):
+        results = []
+        for pipeline in self.dryccfile.get('pipeline', {}).values():
+            if pipeline['ptype'] in ptypes:
+                results.append(pipeline)
+        return results
+
+    def create_release(self, user, *args, **kwargs):
+        latest_release = Release.latest(self.app)
         try:
             new_release = latest_release.new(
                 user,
                 build=self,
-                config=latest_release.config,
-                canary=len(app_settings.canaries) > 0,
+                config=self._get_or_create_config(),
             )
-            self.app.deploy(new_release)
+            if self.app.appsettings_set.latest().autodeploy:
+                new_release.deploy(force_deploy=False)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
+            latest_version = self.app.release_set.latest().version
             if ('new_release' not in locals() and
                     self.app.release_set.latest().version == latest_version+1):
                 new_release = self.app.release_set.latest()
-            if 'new_release' in locals():
+                new_release.state = "crashed"
                 new_release.failed = True
-                new_release.summary = "{} deployed {} which failed".format(self.owner, str(self.uuid)[:7])  # noqa
+                if new_release.summary:
+                    new_release.summary += " "
+                new_release.summary += "{} deployed {} which failed".format(
+                    user, str(self.uuid)[:7])
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
-                new_release.save()
-            else:
+                # avoid overwriting other fields
+                new_release.save(update_fields=["state", "failed", "summary", "exception"])
+            if 'new_release' not in locals():
                 self.delete()
-
             raise DryccException(str(e)) from e
 
-    def save(self, **kwargs):
-        previous_release = self.app.release_set.filter(failed=False).latest()
-
-        if (
-            settings.DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING is True and
-            # previous release had a Procfile and the current one does not
-            (
-                previous_release.build is not None and
-                len(previous_release.build.procfile) > 0 and
-                len(self.procfile) == 0
-            )
-        ):
-            # Reject deployment
-            raise Conflict(
-                'Last deployment had a Procfile but is missing in this deploy. '
-                'For a successful deployment provide a Procfile.'
-            )
-
-        # See if processes are permitted to be removed
-        remove_procs = (
-            # If set to True then contents of Procfile does not affect the outcome
-            settings.DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE is True or
-            # previous release had a Procfile and the current one does as well
-            (
-                previous_release.build is not None and
-                len(previous_release.build.procfile) > 0 and
-                len(self.procfile) > 0
-            )
-        )
-
-        # spin down any proc type removed between the last procfile and the newest one
-        if remove_procs and previous_release.build is not None:
-            removed = {}
-            for proc in previous_release.build.procfile:
-                if proc not in self.procfile:
-                    # Scale proc type down to 0
-                    removed[proc] = 0
-
-            self.app.scale(self.owner, removed)
-
-        # make sure the latest build has procfile if the intent is to
-        # allow empty Procfile without removals
-        if (
-            settings.DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE is False and
-            previous_release.build is not None and
-            len(previous_release.build.procfile) > 0 and
-            len(self.procfile) == 0
-        ):
-            self.procfile = previous_release.build.procfile
-
-        return super(Build, self).save(**kwargs)
-
     def __str__(self):
         return "{0}-{1}".format(self.app.id, str(self.uuid)[:7])
+
+    def _get_or_create_config(self):
+        """
+        dryccfile to config
+        """
+        if not self.dryccfile:
+            return self.app.release_set.filter(failed=False).latest().config
+        # create config from dryccfile
+        values, values_ref = [], {}
+        for group, envs in self.dryccfile.get('config', {}).items():
+            for key, value in envs.items():
+                values.append({"name": key, "group": group, "value": value})
+        for pipeline in self.dryccfile.get('pipeline', {}).values():
+            if 'env' in pipeline or 'config' in pipeline:
+                for key, value in pipeline.get('env', {}).items():
+                    values.append({"name": key, "ptype": pipeline['ptype'], "value": value})
+                for config_ref in pipeline.get('config', []):
+                    if pipeline['ptype'] not in values_ref:
+                        values_ref[pipeline['ptype']] = [config_ref]
+                    else:
+                        values_ref[pipeline['ptype']].append(config_ref)
+        config = Config(app=self.app, values=values, values_refs=values_ref)
+        config.save(ignore_update_fields=["values", "values_refs"])
+        return config
diff --git a/rootfs/api/models/certificate.py b/rootfs/api/models/certificate.py
index 6498106d5..15778f0aa 100644
--- a/rootfs/api/models/certificate.py
+++ b/rootfs/api/models/certificate.py
@@ -1,10 +1,8 @@
 import logging
-from OpenSSL import crypto
-from pyasn1.codec.der import decoder as der_decoder
-from pyasn1.type import univ, constraint
-from ndg.httpsclient.ssl_peer_verification import SUBJ_ALT_NAME_SUPPORT
-from ndg.httpsclient.subj_alt_name import SubjectAltName as BaseSubjectAltName
-from datetime import datetime
+
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.x509.oid import ExtensionOID, NameOID
 from pytz import utc
 
 from django.shortcuts import get_object_or_404
@@ -13,7 +11,7 @@
 from django.contrib.auth import get_user_model
 from rest_framework.exceptions import ValidationError
 from api.utils import validate_label
-from api.exceptions import AlreadyExists, ServiceUnavailable
+from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 from .base import AuditedModel
 from .domain import Domain
@@ -22,59 +20,57 @@
 logger = logging.getLogger(__name__)
 
 
-# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
-class SubjectAltName(BaseSubjectAltName):
-    '''ASN.1 implementation for subjectAltNames support'''
+def get_subj_alt_name(peer_cert):
+    try:
+        san_extension = peer_cert.extensions.get_extension_for_oid(
+            ExtensionOID.SUBJECT_ALTERNATIVE_NAME
+        )
+    except x509.ExtensionNotFound:
+        return []
 
-    # There is no limit to how many SAN certificates a certificate may have,
-    #   however this needs to have some limit so we'll set an arbitrarily high
-    #   limit.
-    sizeSpec = univ.SequenceOf.sizeSpec + \
-        constraint.ValueSizeConstraint(1, 1024)
+    return san_extension.value.get_values_for_type(x509.DNSName)
 
 
-# Note: This is a slightly bug-fixed version of same from ndg-httpsclient.
-def get_subj_alt_name(peer_cert):
-    # Search through extensions
-    dns_name = []
-    if not SUBJ_ALT_NAME_SUPPORT:
-        return dns_name
-
-    general_names = SubjectAltName()
-    for i in range(peer_cert.get_extension_count()):
-        ext = peer_cert.get_extension(i)
-        ext_name = ext.get_short_name()
-        if ext_name != b'subjectAltName':
-            continue
-
-        # PyOpenSSL returns extension data in ASN.1 encoded form
-        ext_dat = ext.get_data()
-        decoded_dat = der_decoder.decode(ext_dat,
-                                         asn1Spec=general_names)
-
-        for name in decoded_dat:
-            if not isinstance(name, SubjectAltName):
-                continue
-            for entry in range(len(name)):
-                component = name.getComponentByPosition(entry)
-                if component.getName() != 'dNSName':
-                    continue
-                dns_name.append(str(component.getComponent()))
-
-    return dns_name
+def public_keys_match(cert_public_key, private_key):
+    def _public_key_der(key):
+        """Serialize any public key to DER SubjectPublicKeyInfo bytes."""
+        return key.public_bytes(
+            serialization.Encoding.DER,
+            serialization.PublicFormat.SubjectPublicKeyInfo,
+        )
+
+    return _public_key_der(cert_public_key) == _public_key_der(
+        private_key.public_key()
+    )
+
+
+def get_certificate_not_valid_after(certificate):
+    if hasattr(certificate, 'not_valid_after_utc'):
+        return certificate.not_valid_after_utc
+
+    return certificate.not_valid_after.replace(tzinfo=utc)
+
+
+def get_certificate_not_valid_before(certificate):
+    if hasattr(certificate, 'not_valid_before_utc'):
+        return certificate.not_valid_before_utc
+
+    return certificate.not_valid_before.replace(tzinfo=utc)
 
 
 def validate_certificate(value):
     try:
-        return crypto.load_certificate(crypto.FILETYPE_PEM, value)
-    except crypto.Error as e:
+        certificate_bytes = value.encode('utf-8') if isinstance(value, str) else value
+        return x509.load_pem_x509_certificate(certificate_bytes)
+    except (TypeError, ValueError) as e:
         raise ValidationError('Could not load certificate: {}'.format(e))
 
 
 def validate_private_key(value):
     try:
-        return crypto.load_privatekey(crypto.FILETYPE_PEM, value)
-    except crypto.Error as e:
+        private_key_bytes = value.encode('utf-8') if isinstance(value, str) else value
+        return serialization.load_pem_private_key(private_key_bytes, password=None)
+    except (TypeError, ValueError) as e:
         raise ValidationError('Could not load private key: {}'.format(e))
 
 
@@ -87,12 +83,8 @@ def validate_cert_pair(certificate, private_key):
         # The certificate and key should already have been validated
         raise SuspiciousOperation(e)
 
-    if pkey.type() == crypto.TYPE_RSA:
-        # Compare modulus n, to the factors p and q
-        priv_numbers = pkey.to_cryptography_key().private_numbers()
-        pub_modulus = cert.get_pubkey().to_cryptography_key().public_numbers().n
-        if pub_modulus != (priv_numbers.p * priv_numbers.q):
-            raise ValidationError('Certificate and private key do not match!')
+    if not public_keys_match(cert.public_key(), pkey):
+        raise ValidationError('Certificate and private key do not match!')
 
     # Return tuple if everything went ok
     return (cert, pkey)
@@ -102,8 +94,8 @@ class Certificate(AuditedModel):
     """
     Public and private key pair used to secure application traffic at the router.
     """
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
-    name = models.CharField(max_length=253, unique=True, validators=[validate_label])
+    app = models.ForeignKey('App', on_delete=models.CASCADE)
+    name = models.CharField(max_length=253, validators=[validate_label])
     # there is no upper limit on the size of an x.509 certificate
     certificate = models.TextField(validators=[validate_certificate])
     key = models.TextField(validators=[validate_private_key])
@@ -121,6 +113,7 @@ class Certificate(AuditedModel):
 
     class Meta:
         ordering = ['name', 'common_name', 'expires']
+        unique_together = ('app', 'name')
 
     @property
     def domains(self):
@@ -130,6 +123,10 @@ def domains(self):
 
         return domains
 
+    @property
+    def certname(self):
+        return '%s-certificate' % self.name
+
     def __str__(self):
         return self.name
 
@@ -138,34 +135,26 @@ def save(self, *args, **kwargs):
         certificate, _ = validate_cert_pair(self.certificate, self.key)
 
         if not self.common_name:
-            self.common_name = certificate.get_subject().CN
+            common_names = certificate.subject.get_attributes_for_oid(NameOID.COMMON_NAME)
+            self.common_name = common_names[0].value if common_names else None
 
         # Grab expire date of the certificate
         if not self.expires:
-            # https://pyopenssl.readthedocs.org/en/latest/api/crypto.html#OpenSSL.crypto.X509.get_notAfter
-            # Convert bytes to string
-            timestamp = certificate.get_notAfter().decode(encoding='UTF-8')
-            # convert openssl's expiry date format to Django's DateTimeField format
-            self.expires = datetime.strptime(timestamp, '%Y%m%d%H%M%SZ').replace(tzinfo=utc)
+            self.expires = get_certificate_not_valid_after(certificate)
 
         # Grab the start date of the certificate
         if not self.starts:
-            # https://pyopenssl.readthedocs.org/en/latest/api/crypto.html#OpenSSL.crypto.X509.get_notBefore
-            # Convert bytes to string
-            timestamp = certificate.get_notBefore().decode(encoding='UTF-8')
-            # convert openssl's starts date format to Django's DateTimeField format
-            self.starts = datetime.strptime(timestamp, '%Y%m%d%H%M%SZ').replace(tzinfo=utc)
+            self.starts = get_certificate_not_valid_before(certificate)
 
         # process issuers - separate each key/value with a slash
-        issuer = certificate.get_issuer().get_components()
-        self.issuer = '/' + '/'.join('%s=%s' % (key.decode(encoding='UTF-8'), value.decode(encoding='UTF-8')) for key, value in issuer)  # noqa
+        self.issuer = certificate.issuer.rfc4514_string()
 
         # process subject - separate each key/value with a slash
-        subject = certificate.get_subject().get_components()
-        self.subject = '/' + '/'.join('%s=%s' % (key.decode(encoding='UTF-8'), value.decode(encoding='UTF-8')) for key, value in subject)  # noqa
+        self.subject = certificate.subject.rfc4514_string()
 
         # public fingerprint of certificate
-        self.fingerprint = certificate.digest('sha256').decode()
+        fingerprint = certificate.fingerprint(hashes.SHA256()).hex().upper()
+        self.fingerprint = ':'.join(fingerprint[i:i + 2] for i in range(0, len(fingerprint), 2))
 
         # SubjectAltName from the certificate - return a list
         self.san = get_subj_alt_name(certificate)
@@ -186,8 +175,6 @@ def delete(self, *args, **kwargs):
     def attach(self, *args, **kwargs):
         # add the certificate to the domain
         domain = get_object_or_404(Domain, domain=kwargs['domain'])
-        if domain.certificate is not None:
-            raise AlreadyExists("Domain already has a certificate attached to it")
         # create in kubernetes
         self.attach_in_kubernetes(domain)
         domain.certificate = self
@@ -197,24 +184,23 @@ def attach_in_kubernetes(self, domain):
         """Creates the certificate as a kubernetes secret"""
         # only create if it exists - We raise an exception when a secret doesn't exist
         try:
-            name = '%s-certificate' % self.name
             namespace = domain.app.id
             data = {
                 'tls.crt': self.certificate,
                 'tls.key': self.key
             }
 
-            secret = self._scheduler.secret.get(namespace, name).json()['data']
+            secret = self.scheduler.secret.get(namespace, self.certname).json()['data']
         except KubeException:
-            self._scheduler.secret.create(namespace, name, data)
+            self.scheduler.secret.create(namespace, self.certname, data)
         else:
             # update cert secret to the TLS Ingress format if required
             if secret != data:
                 try:
-                    self._scheduler.secret.update(namespace, name, data)
+                    self.scheduler.secret.update(namespace, self.certname, data)
                 except KubeException as e:
                     msg = 'There was a problem updating the certificate secret ' \
-                          '{} for {}'.format(name, namespace)
+                          '{} for {}'.format(self.certname, namespace)
                     raise ServiceUnavailable(msg) from e
 
     def detach(self, *args, **kwargs):
@@ -223,14 +209,15 @@ def detach(self, *args, **kwargs):
         domain.certificate = None
         domain.save()
 
-        name = '%s-certificate' % self.name
         namespace = domain.app.id
 
         # only delete if it exists and if no other domains depend on secret
         if len(self.domains) == 0:
             try:
                 # We raise an exception when a secret doesn't exist
-                self._scheduler.secret.get(namespace, name)
-                self._scheduler.secret.delete(namespace, name)
+                self.scheduler.secret.get(namespace, self.certname)
+                self.scheduler.secret.delete(namespace, self.certname)
             except KubeException as e:
-                raise ServiceUnavailable("Could not delete certificate secret {} for application {}".format(name, namespace)) from e  # noqa
+                raise ServiceUnavailable(
+                    "Could not delete certificate secret {} for application {}".format(
+                        self.certname, namespace)) from e
diff --git a/rootfs/api/models/config.py b/rootfs/api/models/config.py
index 257bbd149..51bc0cf9f 100644
--- a/rootfs/api/models/config.py
+++ b/rootfs/api/models/config.py
@@ -1,11 +1,8 @@
-import json
-import math
 import logging
-from decimal import Decimal
-from django.conf import settings
+from functools import partial
 from django.db import models
 from django.contrib.auth import get_user_model
-from api.utils import unit_to_bytes, unit_to_millicpu
+from api.utils import dict_diff
 from api.exceptions import DryccException, UnprocessableEntity
 from .release import Release
 from .base import UuidAuditedModel
@@ -19,16 +16,17 @@ class Config(UuidAuditedModel):
     Set of configuration values applied as environment variables
     during runtime execution of the Application.
     """
+    ptype_fields = ("lifecycle", "tags", "limits", "values_refs", "healthcheck",
+                    "termination_grace_period", "registry")
+    allof_fields = ("values", ) + ptype_fields
 
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    values = models.JSONField(default=dict, blank=True)
-    memory = models.JSONField(default=dict, blank=True)
-    lifecycle_post_start = models.JSONField(default=dict, blank=True)
-    lifecycle_pre_stop = models.JSONField(default=dict, blank=True)
-    cpu = models.JSONField(default=dict, blank=True)
+    values = models.JSONField(default=list, blank=True)
+    values_refs = models.JSONField(default=dict, blank=True)
     tags = models.JSONField(default=dict, blank=True)
+    limits = models.JSONField(default=dict, blank=True)
     registry = models.JSONField(default=dict, blank=True)
+    lifecycle = models.JSONField(default=dict, blank=True)
     healthcheck = models.JSONField(default=dict, blank=True)
     termination_grace_period = models.JSONField(default=dict, blank=True)
 
@@ -40,228 +38,243 @@ class Meta:
     def __str__(self):
         return "{}-{}".format(self.app.id, str(self.uuid)[:7])
 
-    def _migrate_legacy_healthcheck(self):
-        """
-        Get all healthchecks options together for use in scheduler
-        """
-        # return if no legacy healthcheck is found
-        if 'HEALTHCHECK_URL' not in self.values.keys():
-            return
-
-        path = self.values.get('HEALTHCHECK_URL', '/')
-        timeout = int(self.values.get('HEALTHCHECK_TIMEOUT', 50))
-        delay = int(self.values.get('HEALTHCHECK_INITIAL_DELAY', 50))
-        period_seconds = int(self.values.get('HEALTHCHECK_PERIOD_SECONDS', 10))
-        success_threshold = int(self.values.get('HEALTHCHECK_SUCCESS_THRESHOLD', 1))
-        failure_threshold = int(self.values.get('HEALTHCHECK_FAILURE_THRESHOLD', 3))
-
-        self.healthcheck['web'] = {}
-        self.healthcheck['web']['livenessProbe'] = {
-            'initialDelaySeconds': delay,
-            'timeoutSeconds': timeout,
-            'periodSeconds': period_seconds,
-            'successThreshold': success_threshold,
-            'failureThreshold': failure_threshold,
-            'httpGet': {
-                'path': path,
-            }
-        }
-
-        self.healthcheck['web']['readinessProbe'] = {
-            'initialDelaySeconds': delay,
-            'timeoutSeconds': timeout,
-            'periodSeconds': period_seconds,
-            'successThreshold': success_threshold,
-            'failureThreshold': failure_threshold,
-            'httpGet': {
-                'path': path,
-            }
-        }
-
-        # Unset all the old values
-        self.values = {k: v for k, v in self.values.items() if not k.startswith('HEALTHCHECK_')}
-
-    def get_healthcheck(self):
-        if (
-            'livenessProbe' in self.healthcheck.keys() or
-            'readinessProbe' in self.healthcheck.keys()
-        ):
-            return {'web': self.healthcheck}
-        return self.healthcheck
-
-    def _set_cpu_memory(self):
-        """
-        According to settings.KUBERNETES_CPU_MEMORY_RATIO corrects cpu and memory
-        """
-        radio = settings.KUBERNETES_CPU_MEMORY_RATIO
-        limit_min_cpu = settings.KUBERNETES_LIMITS_MIN_CPU
-        limit_max_cpu = settings.KUBERNETES_LIMITS_MAX_CPU
-        limit_min_memory = Decimal(settings.KUBERNETES_LIMITS_MIN_MEMORY * math.pow(1024, 2))
-        limit_max_memory = Decimal(settings.KUBERNETES_LIMITS_MAX_MEMORY * math.pow(1024, 2))
-        memory_cpu_min_radio = Decimal(unit_to_bytes(radio[0])) / Decimal(unit_to_millicpu('1'))
-        memory_cpu_max_radio = Decimal(unit_to_bytes(radio[1])) / Decimal(unit_to_millicpu('1'))
-        cpu_memory_min_radio = Decimal(unit_to_millicpu('1')) / Decimal(unit_to_bytes(radio[1]))
-        cpu_memory_max_radio = Decimal(unit_to_millicpu('1')) / Decimal(unit_to_bytes(radio[0]))
-        for container_type in set(
-                self.app.structure.keys()).union(set(self.cpu)).union(set(self.memory)):
-            if container_type in self.cpu:
-                cpu = unit_to_millicpu(self.cpu[container_type])
-                min_memory = cpu * memory_cpu_min_radio
-                min_memory = limit_min_memory if min_memory < limit_min_memory else min_memory
-                max_memory = cpu * memory_cpu_max_radio
-                max_memory = limit_max_memory if max_memory > limit_max_memory else max_memory
-                if self.memory.get(container_type):
-                    memory = unit_to_bytes(self.memory.get(container_type))
-                    if memory < min_memory:
-                        memory = min_memory
-                    elif memory > max_memory:
-                        memory = max_memory
-                else:
-                    memory = min_memory
-                if memory % Decimal(math.pow(1024, 3)) == 0:
-                    self.memory[container_type] = f'{round(memory / Decimal(math.pow(1024, 3)))}G'
-                else:
-                    self.memory[container_type] = f'{round(memory / Decimal(math.pow(1024, 2)))}M'
-            elif container_type in self.memory:
-                memory = Decimal(unit_to_bytes(self.memory[container_type]))
-                if container_type not in self.cpu:
-                    min_cpu = memory * cpu_memory_min_radio
-                    min_cpu = limit_min_cpu if min_cpu < limit_min_cpu else min_cpu
-                    max_cpu = memory * cpu_memory_max_radio
-                    max_cpu = limit_max_cpu if max_cpu > limit_max_cpu else max_cpu
-                    cpu = max_cpu if min_cpu < 1000 else min_cpu
-                    if cpu % 1000 == 0:
-                        self.cpu[container_type] = f'{round(cpu / 1000)}'
-                    else:
-                        self.cpu[container_type] = f'{round(cpu)}m'
-            else:
-                self.cpu[container_type] = f"{settings.KUBERNETES_LIMITS_MIN_CPU}m"
-                self.memory[container_type] = f"{settings.KUBERNETES_LIMITS_MIN_MEMORY}M"
-
     def previous(self):
         """
-        Return the previous Release to this one.
+        Return the previous Config to this one.
 
-        :return: the previous :class:`Release`, or None
+        :return: the previous :class:`Config`, or None
         """
         configs = self.app.config_set
         if self.pk:
             configs = configs.exclude(pk=self.pk)
 
         try:
-            # Get the Release previous to this one
-            prev_release = configs.latest()
-        except Release.DoesNotExist:
-            prev_release = None
-        return prev_release
+            # Get the Config previous to this one
+            prev_config = configs.latest()
+        except Config.DoesNotExist:
+            prev_config = None
+        return prev_config
 
-    def set_registry(self):
-        # lower case all registry options for consistency
-        self.registry = {key.lower(): value for key, value in self.registry.copy().items()}
+    def envs(self, ptype):
+        envs = {}
+        # group env
+        groups = ['global']
+        groups.extend(self.values_refs.get(ptype, []))
+        for group in groups:
+            for value in self.values:
+                if group == value.get('group'):
+                    envs[value['name']] = value['value']
+        # ptype env, repetition will replace the previous env
+        for value in self.values:
+            if value.get('ptype') == ptype:
+                envs[value['name']] = value['value']
+        return envs
 
-        # PORT must be set if private registry is being used
-        if self.registry and self.values.get('PORT', None) is None:
-            # only thing that can get past post_save in the views
-            raise DryccException(
-                'PORT needs to be set in the config '
-                'when using a private registry')
+    def diff(self, old_config):
+        def flat_values(values):
+            ptype_envs, group_envs = {}, {}
+            for value in values:
+                ptype, group = value.get('ptype'), value.get('group')
+                if ptype:
+                    ptype_envs[ptype] = ptype_envs.get(ptype, {})
+                    ptype_envs[ptype][value['name']] = value['value']
+                if group:
+                    group_envs[group] = group_envs.get(group, {})
+                    group_envs[group][value['name']] = value['value']
+            return ptype_envs, group_envs
 
-    def set_tags(self, previous_config):
-        """verify the tags exist on any nodes as labels"""
-        if not self.tags:
-            if settings.DRYCC_DEFAULT_CONFIG_TAGS:
-                try:
-                    tags = json.loads(settings.DRYCC_DEFAULT_CONFIG_TAGS)
-                    self.tags = tags
-                except json.JSONDecodeError as e:
-                    logger.exception(e)
-                    return
-            else:
-                return
+        old_config = old_config if old_config else self.previous()
+        # ptype field diff
+        result = {}
+        for field in self.ptype_fields:
+            new_value = getattr(self, field)
+            old_value = getattr(old_config, field)
+            result[field] = dict_diff(new_value, old_value)
+        values_diff = {}
+        (new_ptype_envs, new_group_envs), (old_ptype_envs, old_group_envs) = (
+            flat_values(self.values), flat_values(old_config.values))
+        # diff ptype env
+        for ptype in set(new_ptype_envs.keys()).union(old_ptype_envs.keys()):
+            values_diff.update(
+                dict_diff(new_ptype_envs.get(ptype, {}), old_ptype_envs.get(ptype, {})))
+        # diff group env
+        for group in set(new_group_envs.keys()).union(old_group_envs.keys()):
+            values_diff.update(
+                dict_diff(new_group_envs.get(group, {}), old_group_envs.get(group, {})))
+        result["values"] = values_diff
+        return result
 
-        # Get all nodes with label selectors
-        nodes = self._scheduler.node.get(labels=self.tags).json()
-        if nodes['items']:
-            return
+    def diff_ptypes(self, old_config, include_ptypes):
+        old_config = old_config if old_config else self.previous()
+        ptypes = set()
+        for ptype in include_ptypes:
+            for field in self.ptype_fields:
+                new_value = getattr(self, field).get(ptype, None)
+                old_value = getattr(old_config, field).get(ptype, None)
+                if (new_value or old_value) and new_value != old_value:
+                    ptypes.add(ptype)
+            new_env = self.envs(ptype)
+            old_env = old_config.envs(ptype)
+            if new_env != old_env:
+                ptypes.add(ptype)
+        return ptypes
 
-        labels = ['{}={}'.format(key, value) for key, value in self.tags.items()]
-        message = 'No nodes matched the provided labels: {}'.format(', '.join(labels))
+    def save(self, ignore_update_fields=None, *args, **kwargs):
+        """merge the old config with the new"""
+        try:
+            # Get config from the latest available release
+            latest_releases = Release.latest(self.app)
+            if latest_releases:
+                previous_config = latest_releases.config
+            else:
+                # If that doesn't exist then fallback on app config
+                # usually means a totally new app
+                previous_config = self.app.config_set.latest()
+            for field in self.allof_fields:
+                if ignore_update_fields is None or field not in ignore_update_fields:
+                    self.merge_field(field, previous_config)
+        except Config.DoesNotExist:
+            self._update_tags(previous_config={'tags': {}})
+        return super(Config, self).save(*args, **kwargs)
 
-        # Find out if there are any other tags around
-        old_tags = getattr(previous_config, 'tags')
-        if old_tags:
-            old = ['{}={}'.format(key, value) for key, value in old_tags.items()]
-            new = set(labels) - set(old)
-            if new:
-                message += ' - Addition of {} is the cause'.format(', '.join(new))
+    def merge_field(self, field, old_config, *args, **kwargs):
+        getattr(
+            self,
+            "_update_%s" % field,
+            partial(self._update_field, field)
+        )(old_config, *args, **kwargs)
 
-        raise DryccException(message)
+    def _update_field(self, field, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, field, {}).copy().items()
+            if k not in replace_ptypes
+        }
+        new_data = getattr(self, field, {}).copy()
+        # remove config keys if a null value is provided
+        for key, value in new_data.items():
+            if value is None:
+                # error if unsetting non-existing key
+                if key not in data:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(key, field))
+                data.pop(key)
+            else:
+                data[key] = self._merge_data(field, data.get(key, {}), value)
+        setattr(self, field, data)
+
+    def _update_values(self, previous_config, replace_ptypes=[], replace_groups=[]):
+        data = [
+            item for item in getattr(previous_config, 'values', []).copy()
+            if item.get('ptype') not in replace_ptypes and item.get('group') not in replace_groups
+        ]
+        new_data = getattr(self, 'values', []).copy()
+        for new_item in new_data:
+            added = True
+            for index, item in enumerate(data):
+                if (
+                    item['name'] == new_item['name'] and
+                    item.get('ptype') == new_item.get('ptype') and
+                    item.get('group') == new_item.get('group')
+                ):
+                    data.pop(index)
+                    if not new_item['value']:
+                        added = False
+                    else:  # force to string
+                        new_item['value'] = str(new_item['value'])
+                    break
+            if added and new_item['value'] is not None:
+                data.append(new_item)
+        setattr(self, 'values', data)
 
-    def set_healthcheck(self, previous_config):
-        data = getattr(previous_config, 'healthcheck', {}).copy()
-        new_data = getattr(self, 'healthcheck', {}).copy()
-        # update the config data for healthcheck if they are not
-        # present for per proctype
-        # TODO: This is required for backward compatibility and can be
-        # removed in next major version change.
-        if 'livenessProbe' in data.keys() or 'readinessProbe' in data.keys():
-            data = {'web': data.copy()}
-        if 'livenessProbe' in new_data.keys() or 'readinessProbe' in new_data.keys():  # noqa
-            new_data = {'web': new_data.copy()}
+    def _update_values_refs(self, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, 'values_refs', {}).copy().items()
+            if k not in replace_ptypes
+        }
+        new_data = getattr(self, 'values_refs', {}).copy()
+        # remove config keys if a null value is provided
+        for ptype, values in new_data.items():
+            if not values:
+                # error if unsetting non-existing key
+                if ptype not in data:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(ptype, 'values_refs'))
+                data.pop(ptype)
+            else:
+                values_refs = data.get(ptype, [])
+                values_refs.extend(values)
+                data[ptype] = list(set(values_refs))
+        setattr(self, 'values_refs', data)
 
+    def _merge_data(self, field, data, new_data):
         # remove config keys if a null value is provided
         for key, value in new_data.items():
             if value is None:
                 # error if unsetting non-existing key
                 if key not in data:
-                    raise UnprocessableEntity('{} does not exist under {}'.format(key, 'healthcheck'))  # noqa
+                    raise UnprocessableEntity('{} does not exist under {}'.format(key, field))  # noqa
                 data.pop(key)
             else:
-                for probeType, probe in value.items():
-                    if probe is None:
-                        # error if unsetting non-existing key
-                        if key not in data or probeType not in data[key].keys():
-                            raise UnprocessableEntity('{} does not exist under {}'.format(key, 'healthcheck'))  # noqa
-                        data[key].pop(probeType)
-                    else:
-                        if key not in data:
-                            data[key] = {}
-                        data[key][probeType] = probe
-        setattr(self, 'healthcheck', data)
+                data[key] = value
+        return data
 
-    def save(self, **kwargs):
-        """merge the old config with the new"""
-        try:
-            # Get config from the latest available release
-            try:
-                previous_config = self.app.release_set.filter(failed=False).latest().config
-            except Release.DoesNotExist:
-                # If that doesn't exist then fallback on app config
-                # usually means a totally new app
-                previous_config = self.app.config_set.latest()
-
-            for attr in ['cpu', 'memory', 'tags', 'registry', 'values',
-                         'lifecycle_post_start', 'lifecycle_pre_stop',
-                         'termination_grace_period']:
-                data = getattr(previous_config, attr, {}).copy()
-                new_data = getattr(self, attr, {}).copy()
+    def _update_tags(self, previous_config, replace_ptypes=[]):
+        """verify the tags exist on any nodes as labels"""
+        data = {
+            k: v for k, v in getattr(previous_config, 'tags', {}).copy().items()
+            if k not in replace_ptypes
+        }
+        new_data = getattr(self, 'tags', {}).copy()
+        # remove config keys if a null value is provided
+        for ptype, values in new_data.items():
+            if not values:
+                # error if unsetting non-existing key
+                if ptype not in data:
+                    raise UnprocessableEntity(
+                        '{} does not exist under {}'.format(ptype, 'tags'))
+                data.pop(ptype)
+            else:
+                if not self.scheduler.node.get(labels=values).json()['items']:
+                    labels = ['{}={}'.format(key, value) for key, value in values.items()]
+                    message = 'No nodes matched the provided labels: {}'.format(', '.join(labels))
+                    # Find out if there are any other tags around
+                    old_tags = previous_config.tags.get(ptype, {})
+                    if old_tags:
+                        old = ['{}={}'.format(key, value) for key, value in old_tags.items()]
+                        new = set(labels) - set(old)
+                        if new:
+                            message += ' - Addition of {} is the cause'.format(', '.join(new))
+                    raise DryccException(message)
+                data[ptype] = self._merge_data(
+                    'tags', data.get(ptype, {}), values)
+        setattr(self, 'tags', data)
 
-                # remove config keys if a null value is provided
-                for key, value in new_data.items():
-                    if value is None:
-                        # error if unsetting non-existing key
-                        if key not in data:
-                            raise UnprocessableEntity('{} does not exist under {}'.format(key, attr))  # noqa
-                        data.pop(key)
-                    else:
-                        data[key] = value
-                setattr(self, attr, data)
-            self._set_cpu_memory()
-            self.set_healthcheck(previous_config)
-            self._migrate_legacy_healthcheck()
-            self.set_registry()
-            self.set_tags(previous_config)
-        except Config.DoesNotExist:
-            self.set_tags({'tags': {}})
+    def _update_limits(self, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, 'limits', {}).copy().items()
+            if k not in replace_ptypes
+        }
+        new_data = getattr(self, 'limits', {}).copy()
+        # check procfile
+        for ptype, value in new_data.items():
+            if value is None:
+                if ptype in self.app.ptypes:
+                    raise UnprocessableEntity(
+                        "the %s has already been used and cannot be deleted" % ptype)
+        self._merge_data('limits', data, new_data)
+        setattr(self, 'limits', data)
 
-        return super(Config, self).save(**kwargs)
+    def _update_termination_grace_period(self, previous_config, replace_ptypes=[]):
+        data = {
+            k: v for k, v in getattr(previous_config, 'termination_grace_period', {}).copy().items()  # noqa
+            if k not in replace_ptypes
+        }
+        new_data = getattr(self, 'termination_grace_period', {}).copy()
+        # check procfile
+        for ptype, value in new_data.items():
+            if value is None:
+                if ptype in self.app.ptypes:
+                    raise UnprocessableEntity(
+                        "the %s has already been used and cannot be deleted" % ptype)
+        self._merge_data('termination_grace_period', data, new_data)
+        setattr(self, 'termination_grace_period', data)
diff --git a/rootfs/api/models/domain.py b/rootfs/api/models/domain.py
index 2279a41ac..a1caff610 100644
--- a/rootfs/api/models/domain.py
+++ b/rootfs/api/models/domain.py
@@ -1,13 +1,12 @@
 from django.db import models
 from django.db import transaction
 from django.contrib.auth import get_user_model
-from .base import AuditedModel
+from .base import AuditedModel, PTYPE_MAX_LENGTH
 
 User = get_user_model()
 
 
 class Domain(AuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     domain = models.TextField(
         blank=False, null=False, unique=True,
@@ -21,6 +20,7 @@ class Domain(AuditedModel):
         blank=True,
         null=True
     )
+    ptype = models.CharField(max_length=PTYPE_MAX_LENGTH)
 
     class Meta:
         ordering = ['domain', 'certificate']
diff --git a/rootfs/api/models/gateway.py b/rootfs/api/models/gateway.py
index 62de210a9..c4e68c323 100644
--- a/rootfs/api/models/gateway.py
+++ b/rootfs/api/models/gateway.py
@@ -1,47 +1,41 @@
-import re
 import logging
+import hashlib
+import threading
 from django.db import models
 from django.conf import settings
-from django.shortcuts import get_object_or_404
-from django.contrib.auth import get_user_model
+from django.db.models import Q
+from django.http import Http404
 
+from api.tasks import send_app_log
+from api.utils import validate_label
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
 
-from .base import AuditedModel
+from .base import AuditedModel, DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT
+
 
-User = get_user_model()
 logger = logging.getLogger(__name__)
 
+TLS_PROTOCOLS = ("HTTPS", "TLS")
+HOSTNAME_PROTOCOLS = TLS_PROTOCOLS + ("HTTP", )
+
 
 class Gateway(AuditedModel):
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
-    name = models.CharField(max_length=63, db_index=True)
+    name = models.CharField(max_length=63, db_index=True, validators=[validate_label])
     ports = models.JSONField(default=list)
 
-    def _check_port(self, port, protocol):
-        for item in self.ports:
-            if item["port"] == port:
-                if (item["protocol"] == protocol) or (
-                        item["protocol"] != "UDP" and protocol != "UDP"):
-                    return False
-        return True
-
-    def _get_tls_domain(self, auto_tls):
-        domains = self.app.domain_set.all()
-        if not auto_tls:
-            domains = domains.exclude(certificate=None)
-        return domains
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this service.
 
-    def _get_listener_name(self, port, protocol, index):
-        if protocol in ("TCP", "TLS", "HTTP"):
-            protocol = "TCP"
-        elif protocol in ("HTTPS", ):
-            protocol = "MIX"
-        else:
-            protocol = "UDP"
-        return "-".join([protocol, str(port), str(index)]).lower()
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def add(self, port, protocol):
         # check port
@@ -60,26 +54,37 @@ def remove(self, port, protocol):
         self.ports = ports
         return True, ""
 
+    @property
+    def addresses(self):
+        data = self.scheduler.gateways.get(self.app.id, self.name, ignore_exception=True)
+        if data.status_code != 200:
+            return []
+        addresses = data.json()["status"].get("addresses", [])
+        return addresses
+
     @property
     def listeners(self):
-        listeners = []
         auto_tls = self.app.tls_set.latest().certs_auto_enabled
-        domains = list(self._get_tls_domain(auto_tls))
+        listeners = []
+        domains = list(self.app.domain_set.all())
         for item in self.ports:
             port, protocol = item["port"], item["protocol"]
-            if item["protocol"] in ("TLS", "HTTPS"):
+            if item["protocol"] in HOSTNAME_PROTOCOLS:
                 for domain in domains:
-                    secret_name = (f"{self.app.id}-auto-tls" if
-                                   auto_tls else domain.certificate.name)
-                    listeners.append({
+                    listener = {
                         "allowedRoutes": {"namespaces": {"from": "All"}},
-                        "name": self._get_listener_name(port, protocol, domains.index(domain)),
+                        "name": self._get_listener_name(port, protocol, domains.index(domain) + 1),
                         "port": port,
                         "hostname": domain.domain,
                         "protocol": protocol,
-                        "tls": {"certificateRefs": [{"kind": "Secret", "name": secret_name}]},
-                    })
-            else:
+                    }
+                    secret_name = f"{self.app.id}-auto-tls" if auto_tls else (
+                        domain.certificate.certname if domain.certificate else None)
+                    if secret_name and protocol in TLS_PROTOCOLS:
+                        listener["tls"] = {
+                            "certificateRefs": [{"kind": "Secret", "name": secret_name}]}
+                    listeners.append(listener)
+            if protocol not in TLS_PROTOCOLS:
                 listeners.append({
                     "allowedRoutes": {"namespaces": {"from": "All"}},
                     "name": self._get_listener_name(port, protocol, 0),
@@ -88,53 +93,89 @@ def listeners(self):
                 })
         return listeners
 
-    @property
-    def addresses(self):
-        data = self._scheduler.gateways.get(self.app.id, self.name, ignore_exception=True)
-        if data.status_code != 200:
-            return []
-        addresses = data.json()["status"].get("addresses", [])
-        return addresses
-
     def refresh_to_k8s(self):
+        kwargs = {"listeners": self.listeners, "gateway_class": settings.DRYCC_APP_GATEWAY_CLASS}
+        if self.app.tls_set.latest().certs_auto_enabled:
+            kwargs["annotations"] = {"cert-manager.io/issuer": self.app.id}
         try:
             try:
-                data = self._scheduler.gateways.get(self.app.id, self.name).json()
-                if len(self.listeners) > 0:
-                    self._scheduler.gateways.patch(self.app.id, self.name, **{
-                        "listeners": self.listeners,
-                        "gateway_class": settings.GATEWAY_CLASS,
-                        "version": data["metadata"]["resourceVersion"],
-                    })
+                data = self.scheduler.gateways.get(self.app.id, self.name).json()
+                if len(kwargs["listeners"]) > 0:
+                    kwargs["version"] = data["metadata"]["resourceVersion"]
+                    response = self.scheduler.gateways.patch(self.app.id, self.name, **kwargs)
+                    if response.status_code == 409:
+                        raise ServiceUnavailable(f'Kubernetes gateway could not be patched: {response.status_code} {response.reason}, please retry')  # noqa 
                 else:
                     logger.debug("delete k8s resource when listeners are empty")
-                    self._scheduler.gateways.delete(
+                    self.scheduler.gateways.delete(
                         self.app.id, self.name, ignore_exception=True)
             except KubeException:
-                if len(self.listeners) > 0:
-                    self._scheduler.gateways.create(self.app.id, self.name, **{
-                        "listeners": self.listeners,
-                        "gateway_class": settings.GATEWAY_CLASS,
-                    })
+                if len(kwargs["listeners"]) > 0:
+                    if "version" in kwargs:
+                        kwargs.pop("version")
+                    self.scheduler.gateways.create(self.app.id, self.name, **kwargs)
                 else:
                     logger.debug("skip creating k8s resource when listeners are empty")
         except KubeException as e:
             raise ServiceUnavailable('Kubernetes gateway could not be created') from e
 
+    def change_default_tls(self):
+        if self.name != self.app.id:
+            return False
+        tls_enabled = (self.app.tls_set.latest().certs_auto_enabled and
+                       self.app.domain_set.exists()) or self.app.domain_set.filter(
+                           models.Q(certificate__isnull=False)).exists()
+        if tls_enabled:
+            return self.add(DEFAULT_HTTPS_PORT, "HTTPS")[0]
+        return self.remove(DEFAULT_HTTPS_PORT, "HTTPS")[0]
+
     def save(self, *args, **kwargs):
+        self.change_default_tls()
         super().save(*args, **kwargs)
         self.refresh_to_k8s()
 
     def delete(self, *args, **kwargs):
         try:
-            self._scheduler.gateways.delete(self.app.id, self.name, ignore_exception=False)
+            self.scheduler.gateways.delete(self.app.id, self.name, ignore_exception=False)
         except KubeException:
-            logger.log(
-                msg='Kubernetes gateway cannot be deleted: {}'.format(self.name),
+            self.log(
+                'Kubernetes gateway cannot be deleted: {}'.format(self.name),
                 level=logging.ERROR,
             )
         return super().delete(*args, **kwargs)
 
+    def to_usages(self, timestamp: float):
+        return [{
+            "app_id": str(self.app_id),
+            "workspace": self.app.workspace_id,
+            "name": settings.DRYCC_APP_GATEWAY_CLASS,
+            "type": "gateway",
+            "unit": "number",
+            "usage": 1,
+            "kwargs": {
+                "name": self.name,
+            },
+            "timestamp": int(timestamp),
+            "identifier": hashlib.md5(self.name.encode("utf-8")).hexdigest(),
+        }]
+
+    def _check_port(self, port, protocol):
+        for item in self.ports:
+            if item["port"] == port:
+                if (item["protocol"] == protocol) or (
+                        item["protocol"] != "UDP" and protocol != "UDP"):
+                    return False
+        return True
+
+    def _get_listener_name(self, port, protocol, index):
+        if protocol in ("TCP", "TLS", "HTTP"):
+            protocol = "TCP"
+        elif protocol in ("HTTPS", ):
+            protocol = "MIX"
+        else:
+            protocol = "UDP"
+        return "-".join([protocol, str(port), str(index)]).lower()
+
     class Meta:
         get_latest_by = 'created'
         unique_together = (('app', 'name'), )
@@ -142,6 +183,7 @@ class Meta:
 
 
 class Route(AuditedModel):
+    CACHE = threading.local()
     PROTOCOLS_CHOICES = {
         "TLSRoute": ("TCP", ),
         "TCPRoute": ("TCP", ),
@@ -151,15 +193,24 @@ class Route(AuditedModel):
     }
 
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     kind = models.CharField(max_length=15, choices=[
         (key, '/'.join(value)) for key, value in PROTOCOLS_CHOICES.items()])
     name = models.CharField(max_length=63, db_index=True)
-    port = models.PositiveIntegerField()
     rules = models.JSONField(default=list)
     routable = models.BooleanField(default=True)
     parent_refs = models.JSONField(default=list)
-    procfile_type = models.TextField()
+
+    @property
+    def services(self):
+        key = f"{self.app.id}_{self.name}"
+        if not hasattr(self.CACHE, key):
+            service_names = set()
+            for rule in self.rules:
+                for backend in rule['backendRefs']:
+                    service_names.add(backend['name'])
+            setattr(self.CACHE, key,
+                    [s for s in self.app.service_set.all() if s.name in service_names])
+        return getattr(self.CACHE, key)
 
     @property
     def protocols(self):
@@ -168,71 +219,95 @@ def protocols(self):
         return self.PROTOCOLS_CHOICES[self.kind]
 
     @property
-    def default_rules(self):
-        service = get_object_or_404(self.app.service_set, procfile_type=self.procfile_type)
-        stable_backend_refs, canary_backend_refs = [], []
-        for item in service.ports:
-            if item["port"] == self.port:
-                stable_backend = {
-                    "kind": "Service",
-                    "name": str(service),
-                    "port": item["port"],
-                    "weight": 100,
-                }
-                canary_backend = {
-                    "kind": "Service",
-                    "name": "%s-canary" % str(service),
-                    "port": item["port"],
-                    "weight": 0,
-                }
-                stable_backend_refs.append(stable_backend)
-                canary_backend_refs.append(stable_backend)
-                canary_backend_refs.append(canary_backend)
-        return {
-            "stable": [{"backendRefs": stable_backend_refs}],
-            "canary": [{"backendRefs": canary_backend_refs}],
-        }
+    def hostnames(self):
+        return [domain.domain for domain in self.app.domain_set.filter(
+                ptype__in=[s.ptype for s in self.services])]
 
     @property
-    def current_rules(self):
-        app_settings = self.app.appsettings_set.latest()
-        if self.procfile_type in app_settings.canaries:
-            return self.rules["canary"]
-        return self.rules["stable"]
-
-    def check_rules(self):
-        service = self.app.service_set.filter(
-            procfile_type=self.procfile_type).first()
-        ports = [item["port"] for item in service.ports]
-        for rules in self.rules.values():
-            for rule in rules:
-                for backend_ref in rule["backendRefs"]:
-                    port = backend_ref["port"]
-                    name = re.sub(r'(.*)-canary', r'\1', backend_ref["name"])
-                    if port not in ports or name != str(service):
-                        return False, {"detail": "backendRefs associated with incorrect service"}
+    def cleaned_rules(self):
+        services, rules = self.services, []
+        for rule in self.rules:
+            backend_refs = []
+            for backend_ref in rule["backendRefs"]:
+                for service in services:
+                    ports = [item["port"] for item in service.ports]
+                    if backend_ref["port"] in ports and backend_ref["name"] == service.name:
+                        backend_refs.append(backend_ref)
+            if backend_refs:
+                rule['backendRefs'] = backend_refs
+                rules.append(rule)
+        return rules
+
+    @property
+    def tls_force_hostnames(self):
+        tls = self.app.tls_set.latest()
+        q = Q(ptype__in=[s.ptype for s in self.services])
+        if not tls.certs_auto_enabled:
+            q &= Q(certificate__isnull=False)
+        domains = self.app.domain_set.filter(q)
+        return [domain.domain for domain in domains]
+
+    def check_rules(self, rules):
+        for rule in rules:
+            for backend_ref in rule["backendRefs"]:
+                kind = backend_ref.get("kind", "Service")
+                if kind != "Service":
+                    return False, {"detail": "BackendRef only supports service kind"}
+                has_service = False
+                for service in self.services:
+                    ports = [item["port"] for item in service.ports]
+                    if backend_ref["port"] in ports and backend_ref["name"] == service.name:
+                        has_service = True
+                        break
+                if not has_service:
+                    msg = f"service {backend_ref['name']}:{backend_ref['port']} does not exist"
+                    return False, {"detail": msg}
         return True, ""
 
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this service.
+
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
+
     def refresh_to_k8s(self):
         if self.routable:
             parent_refs, http_parent_refs = self._get_all_parent_refs()
             tls = self.app.tls_set.latest()
-            if tls.https_enforced and self.kind == "HTTPRoute":
+            # requestRedirect only when has tls or certs
+            if tls.https_enforced and self.kind == "HTTPRoute" and self.tls_force_hostnames:
                 self._https_enforced_to_k8s(http_parent_refs)
             elif self.kind == "HTTPRoute":
                 parent_refs.extend(http_parent_refs)
-                self._scheduler.httproute.delete(self.app.id, self._https_redirect_name)
+                self.scheduler.httproute.delete(self.app.id, self._https_redirect_name)
             else:
                 parent_refs.extend(http_parent_refs)
-            self._refresh_to_k8s(self.current_rules, parent_refs)
+            self._refresh_to_k8s(self.rules, parent_refs)
         else:
-            self._scheduler.httproute.delete(self.app.id, self.name)
-            self._scheduler.httproute.delete(self.app.id, self._https_redirect_name)
+            self.scheduler.httproute.delete(self.app.id, self.name)
+            self.scheduler.httproute.delete(self.app.id, self._https_redirect_name)
+
+    def change_default_tls(self):
+        if self.app.id != self.name:
+            return False
+        tls_enabled = (self.app.tls_set.latest().certs_auto_enabled and
+                       self.app.domain_set.exists()) or self.app.domain_set.filter(
+                           models.Q(certificate__isnull=False)).exists()
+        if tls_enabled:
+            return self.attach(self.app.id, DEFAULT_HTTPS_PORT)[0]
+        else:
+            return self.detach(self.app.id, DEFAULT_HTTPS_PORT)[0]
 
     def attach(self, gateway_name, port):
         ok, msg = self._check_parent(gateway_name, port)
         if not ok:
-            return ok, msg
+            return ok, {"detail": msg}
         parent_ref = {"name": gateway_name, "port": port}
         if parent_ref in self.parent_refs:
             return False, {"detail": "gateway and port already exist in this route"}
@@ -249,19 +324,23 @@ def detach(self, gateway_name, port):
         return True, ""
 
     def save(self, *args, **kwargs):
-        ok, msg = self.check_rules()
-        if not ok:
-            raise ValueError(msg)
+        self.change_default_tls()
+        cleaned_rules = self.cleaned_rules
+        if not cleaned_rules:
+            msg = f"route {self.name} no available backend"
+            self.log(msg, level=logging.ERROR)
+            raise Http404(msg)
+        self.rules = cleaned_rules
         super().save(*args, **kwargs)
         self.refresh_to_k8s()
 
     def delete(self, *args, **kwargs):
         try:
-            k8s_route = getattr(self._scheduler, self.kind.lower())
+            k8s_route = getattr(self.scheduler, self.kind.lower())
             k8s_route.delete(self.app.id, self.name, ignore_exception=False)
         except KubeException:
-            logger.log(
-                msg='Kubernetes {} cannot be deleted: {}'.format(self.kind.lower(), self.name),
+            self.log(
+                'Kubernetes {} cannot be deleted: {}'.format(self.kind.lower(), self.name),
                 level=logging.ERROR,
             )
         return super().delete(*args, **kwargs)
@@ -274,65 +353,64 @@ def _check_parent(self, gateway_name, port):
         try:
             gateway = self.app.gateway_set.filter(name=gateway_name).latest()
         except Gateway.DoesNotExist:
-            return False, {"detail": f"this gateway {gateway_name} does not exist"}
+            return False, f"this gateway {gateway_name} does not exist"
         is_listener_allowed = False
         for gateway_port in gateway.ports:
             if port == gateway_port.get("port") and \
                     self.kind.split("Route")[0] in gateway_port.get("protocol"):
                 is_listener_allowed = True
         if not is_listener_allowed:
-            return False, {"detail": f"this gateway does not allow {self.kind} port {port} bind, \nplease add gateway listener first."}  # noqa
+            return False, "listener does not exist, please add gateway listener first."
         for route in self.app.route_set.exclude(app=self.app, name=self.name):
             for parent_ref in route.parent_refs:
                 if parent_ref["name"] == gateway_name and parent_ref["port"] == port:
-                    for protocol in self.protocols:
-                        if protocol in route.protocols:
-                            return False, {"detail": "this listener has already been referenced"}
+                    if not set(route.protocols).issubset(HOSTNAME_PROTOCOLS) and (
+                            set(route.protocols).issubset(self.protocols) or
+                            set(self.protocols).issubset(route.protocols)):
+                        return False, "this listener has already been referenced"
         return True, ""
 
     def _refresh_to_k8s(self, rules, parent_refs):
+        manifest = {
+            "rules": rules,
+            "hostnames": self.hostnames,
+            "parent_refs": parent_refs,
+        }
         try:
-            k8s_route = getattr(self._scheduler, self.kind.lower())
-            hostnames = [domain.domain for domain in self.app.domain_set.all()]
+            k8s_route = getattr(self.scheduler, self.kind.lower())
             try:
                 data = k8s_route.get(self.app.id, self.name).json()
-                k8s_route.patch(self.app.id, self.name, **{
-                    "rules": rules,
-                    "hostnames": hostnames,
-                    "parent_refs": parent_refs,
-                    "version": data["metadata"]["resourceVersion"],
-                })
+                manifest.update({"version": data["metadata"]["resourceVersion"]})
+                k8s_route.patch(self.app.id, self.name, **manifest)
             except KubeException:
-                k8s_route.create(self.app.id, self.name, **{
-                    "rules": rules,
-                    "hostnames": hostnames,
-                    "parent_refs": parent_refs,
-                })
+                k8s_route.create(self.app.id, self.name, **manifest)
         except KubeException as e:
             raise ServiceUnavailable(
                 f'Kubernetes {self.kind.lower()} could not be created') from e
 
     def _https_enforced_to_k8s(self, parent_refs):
-        rules = {
-            "filters": [{
-                "type": "RequestRedirect",
-                "requestRedirect": {"port": 443, "scheme": "https", "statusCode": 301}
-            }]
+        manifest = {
+            "rules": [{
+                "filters": [{
+                    "type": "RequestRedirect",
+                    "requestRedirect": {
+                        "port": DEFAULT_HTTPS_PORT, "scheme": "https", "statusCode": 301
+                    }
+                }]
+            }],
+            "hostnames": self.tls_force_hostnames,
+            "parent_refs": parent_refs,
         }
         try:
             try:
-                data = self._scheduler.httproute.get(
+                data = self.scheduler.httproute.get(
                     self.app.id, self._https_redirect_name).json()
-                self._scheduler.httproute.patch(self.app.id, self._https_redirect_name, **{
-                    "rules": rules,
-                    "parent_refs": parent_refs,
-                    "version": data["metadata"]["resourceVersion"],
-                })
+                manifest.update({"version": data["metadata"]["resourceVersion"]})
+                self.scheduler.httproute.patch(
+                    self.app.id, self._https_redirect_name, **manifest)
             except KubeException:
-                self._scheduler.httproute.create(self.app.id, self._https_redirect_name, **{
-                    "rules": rules,
-                    "parent_refs": parent_refs,
-                })
+                self.scheduler.httproute.create(
+                    self.app.id, self._https_redirect_name, **manifest)
         except KubeException as e:
             raise ServiceUnavailable(
                 f'Kubernetes {self.kind.lower()} could not be created') from e
@@ -342,21 +420,23 @@ def _get_all_parent_refs(self):
         for gateway in self.app.gateway_set.filter(
                 name__in=[item["name"] for item in self.parent_refs]):
             gateways[gateway.name] = gateway
-        parent_refs, http_parent_refs = [], []
+        hostnames, parent_refs, http_parent_refs = self.hostnames, [], []
         for item in self.parent_refs:
             gateway_name, port = item["name"], item["port"]
             if gateway_name not in gateways:
                 continue
             gateway = gateways[gateway_name]
             for listener in gateway.listeners:
-                if listener["port"] == port and listener["protocol"] in self.protocols:
+                hostname = listener.get('hostname', None)
+                if listener["port"] == port and listener["protocol"] in self.protocols and (
+                        hostname is None or hostname in hostnames):
                     parent_ref = {
                         "group": "gateway.networking.k8s.io",
                         "kind": "Gateway",
                         "name": gateway_name,
                         "sectionName": listener["name"],
                     }
-                    if listener["protocol"] == "HTTP" and listener["port"] == 80:
+                    if listener["protocol"] == "HTTP" and listener["port"] == DEFAULT_HTTP_PORT:
                         http_parent_refs.append(parent_ref)
                     else:
                         parent_refs.append(parent_ref)
diff --git a/rootfs/api/models/key.py b/rootfs/api/models/key.py
index b94867e59..08e94cdf8 100644
--- a/rootfs/api/models/key.py
+++ b/rootfs/api/models/key.py
@@ -33,6 +33,9 @@ class Meta:
         verbose_name = 'SSH Key'
         unique_together = (('owner', 'fingerprint'))
         ordering = ['public']
+        indexes = [
+            models.Index(fields=['fingerprint']),
+        ]
 
     def __str__(self):
         return "{}...{}".format(self.public[:18], self.public[-31:])
diff --git a/rootfs/api/models/limit.py b/rootfs/api/models/limit.py
new file mode 100644
index 000000000..a72eda1a4
--- /dev/null
+++ b/rootfs/api/models/limit.py
@@ -0,0 +1,136 @@
+import logging
+from functools import partial
+from django.db import models
+from django.contrib.auth import get_user_model
+from api.utils import validate_json
+from .base import AuditedModel
+
+
+User = get_user_model()
+logger = logging.getLogger(__name__)
+
+spec_memory_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "size": {"type": "string"},
+        "type": {"type": "string"},
+    },
+    "required": ["size", "type"],
+}
+
+spec_cpu_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "cores": {"type": "integer"},
+        "clock": {"type": "string"},
+        "boost": {"type": "string"},
+        "threads": {"type": "integer"},
+    },
+    "required": ["name", "cores", "clock", "boost", "threads"],
+}
+
+spec_gpu_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "tmus": {"type": "integer"},
+        "rops": {"type": "integer"},
+        "cores": {"type": "integer"},
+        "memory": spec_memory_schema,
+    },
+    "required": ["name", "tmus", "rops", "cores"],
+}
+
+spec_features_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "gpu": spec_gpu_schema,
+        "network": {"type": "string"},
+    },
+    "required": ["gpu", "network"],
+}
+
+spec_keywords_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "array",
+    "minItems": 1,
+    "items": {"type": "string"}
+}
+
+plan_features_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "gpu": {"type": "integer"},
+        "network": {"type": "integer"},
+    },
+    "required": ["gpu", "memory"],
+}
+
+plan_limits_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "cpu": {"type": "integer"},
+        "memory": {"type": "integer"},
+    },
+    "required": ["cpu", "memory"],
+}
+
+
+class LimitSpec(AuditedModel):
+    id = models.CharField(max_length=63, primary_key=True)
+    cpu = models.JSONField(
+        validators=[partial(validate_json, schema=spec_cpu_schema)])
+    memory = models.JSONField(
+        validators=[partial(validate_json, schema=spec_memory_schema)])
+    features = models.JSONField(
+        validators=[partial(validate_json, schema=spec_features_schema)])
+    keywords = models.JSONField(
+        validators=[partial(validate_json, schema=spec_keywords_schema)])
+    disabled = models.BooleanField(default=False)
+    priority = models.SmallIntegerField(default=100)
+
+    class Meta:
+        get_latest_by = 'created'
+        ordering = ['-priority']
+
+
+class LimitPlan(AuditedModel):
+    id = models.CharField(max_length=63, primary_key=True)
+    spec = models.ForeignKey(LimitSpec, on_delete=models.PROTECT)
+    cpu = models.SmallIntegerField(default=1)
+    memory = models.SmallIntegerField(default=1)
+    features = models.JSONField(
+        validators=[partial(validate_json, schema=plan_features_schema)])
+    disabled = models.BooleanField(default=False)
+    priority = models.SmallIntegerField(default=100)
+    limits = models.JSONField(
+        validators=[partial(validate_json, schema=plan_limits_schema)])
+    requests = models.JSONField(default=dict)
+    annotations = models.JSONField(default=dict)
+    node_selector = models.JSONField(default=dict)
+    runtime_class_name = models.CharField(max_length=63, default="")
+    pod_security_context = models.JSONField(default=dict)
+    pod_volumes = models.JSONField(default=list)
+    container_security_context = models.JSONField(default=dict)
+    container_volume_mounts = models.JSONField(default=list)
+
+    class Meta:
+        get_latest_by = 'created'
+        ordering = ['priority']
+        indexes = [
+            models.Index(fields=['spec', 'cpu', 'memory']),
+        ]
+
+    def __str__(self):
+        return self.name
+
+    @staticmethod
+    def get_default():
+        return LimitPlan.objects.filter(disabled=False).first()
diff --git a/rootfs/api/models/release.py b/rootfs/api/models/release.py
index cb09bc503..8a9f8c2f8 100644
--- a/rootfs/api/models/release.py
+++ b/rootfs/api/models/release.py
@@ -1,12 +1,21 @@
 import logging
 
+from datetime import datetime, timezone
 from django.conf import settings
 from django.db import models
+from django.db.models import Q
 from django.contrib.auth import get_user_model
-from api.utils import dict_diff
+from django.db.models import F, Func, Value, JSONField
+from api.tasks import run_pipeline, send_app_log
 from api.exceptions import DryccException, AlreadyExists
 from scheduler import KubeHTTPException
-from .base import UuidAuditedModel
+from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
+
+from ..utils import DeployLock, dict_diff
+
+from .base import UuidAuditedModel, PTYPE_WEB
+from .appsettings import AppSettings
+
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
@@ -18,14 +27,19 @@ class Release(UuidAuditedModel):
 
     Releases contain a :class:`Build` and a :class:`Config`.
     """
-
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    STATE_CHOICES = (
+        ("created", "Release created but not deployed"),
+        ("crashed", "Release pipeline runtime crashed"),
+        ("succeed", "Release pipeline runtime succeed"),
+    )
     app = models.ForeignKey('App', on_delete=models.CASCADE)
+    state = models.TextField(choices=STATE_CHOICES, default=STATE_CHOICES[0][0])
     version = models.PositiveIntegerField()
     summary = models.TextField(blank=True, null=True)
     failed = models.BooleanField(default=False)
-    canary = models.BooleanField(default=False)
     exception = models.TextField(blank=True, null=True)
+    conditions = models.JSONField(default=list)
+    deployed_ptypes = models.JSONField(default=list)
 
     config = models.ForeignKey('Config', on_delete=models.CASCADE)
     build = models.ForeignKey('Build', null=True, on_delete=models.CASCADE)
@@ -34,15 +48,93 @@ class Meta:
         get_latest_by = 'created'
         ordering = ['-created']
         unique_together = (('app', 'version'),)
+        indexes = [
+            models.Index(fields=['app', 'failed', 'state']),
+        ]
 
     def __str__(self):
-        return "{0}-v{1}".format(self.app.id, self.version)
+        return "{0}-{1}".format(self.app.id, self.version_name)
 
     @property
-    def image(self):
-        return self.build.image
+    def ptypes(self):
+        if self.build is not None:
+            return self.build.ptypes
+        return [PTYPE_WEB]
+
+    @property
+    def version_name(self):
+        return f'v{self.version}'
+
+    def get_runners(self, ptypes):
+        results = []
+        ptypes = self.ptypes if not ptypes else ptypes
+        for pipeline in self.build.get_pipelines(ptypes):
+            if 'run' in pipeline:
+                image = pipeline['run'].get('image', self.build.get_image(pipeline['ptype']))
+                results.append({
+                    'ptype': pipeline['ptype'],
+                    'image': self.build.get_image(image, default_image=image),
+                    'args': pipeline['run'].get('args', []),
+                    'command': pipeline['run'].get('command', []),
+                    'timeout': pipeline['run'].get('timeout', settings.DRYCC_PILELINE_RUN_TIMEOUT),
+                })
+        return results
+
+    def add_condition(self, **kwargs):
+        if "created" not in kwargs:
+            kwargs["created"] = datetime.now(timezone.utc).strftime(settings.DRYCC_DATETIME_FORMAT)
+        type(self).objects.filter(pk=self.pk).update(
+            conditions=Func(
+                F("conditions"),
+                Value(["0"]),
+                Value(kwargs, JSONField()),
+                function="jsonb_insert",
+            )
+        )
+
+    def get_deploy_image(self, ptype):
+        """
+        Return the kubernetes "container image" to be sent off to the scheduler.
+        """
+        image = self.build.get_pipeline(ptype).get('deploy', {}).get(
+            'image', self.build.get_image(ptype))
+        return self.build.get_image(image, default_image=image)
+
+    def get_deploy_args(self, ptype):
+        """
+        Return the kubernetes "container arguments" to be sent off to the scheduler.
+        """
+        if self.build is not None:
+            if self.build.dryccfile:
+                return self.build.get_pipeline(ptype).get('deploy', {}).get('args', [])
+            else:
+                # dockerfile or container image
+                if self.build.dockerfile or not self.build.sha:
+                    # has profile
+                    if self.build.procfile and ptype in self.build.procfile:
+                        args = self.build.procfile[ptype]
+                        return args.split()
+        return []
+
+    def get_deploy_command(self, ptype):
+        """
+        Return the kubernetes "container command" to be sent off to the scheduler.
+        """
+        return self.build.get_pipeline(ptype).get('deploy', {}).get('command', [])
+
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this application.
 
-    def new(self, user, config, build, summary=None, canary=False):
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
+
+    def new(self, user, config, build, summary=None):
         """
         Create a new application release using the provided Build and Config
         on behalf of a user.
@@ -53,76 +145,94 @@ def new(self, user, config, build, summary=None, canary=False):
         new_version = self.app.release_set.latest().version + 1
         # create new release and auto-increment version
         return Release.objects.create(
-            owner=user, app=self.app, config=config,
-            build=build, version=new_version, summary=summary, canary=canary
+            app=self.app, config=config, build=build, version=new_version, summary=summary
         )
 
-    def get_port(self):
+    def get_port(self, ptype):
         """
         Get application port for a given release. If pulling from private registry
         then use default port or read from ENV var, otherwise attempt to pull from
         the container image
         """
+        return int(self.config.envs(ptype).get('PORT', DEFAULT_CONTAINER_PORT))
+
+    def diff_ptypes(self, ptypes):
+        def _get_full_deploy(release, ptypes):
+            deploy = {}
+            for ptype in ptypes:
+                deploy[ptype] = {
+                    'image': release.get_deploy_image(ptype),
+                    'args': release.get_deploy_args(ptype),
+                    'command': release.get_deploy_command(ptype),
+                }
+            return deploy
+
+        pre_release = self.previous()
+        if pre_release is None:
+            return ptypes
+        changed_ptypes = set(self.config.diff_ptypes(pre_release.config, ptypes))
+        if pre_release.build and self.build:
+            deploy = _get_full_deploy(self, ptypes)
+            pre_deploy = _get_full_deploy(pre_release, ptypes)
+            for value in dict_diff(deploy, pre_deploy).values():
+                changed_ptypes = changed_ptypes.union(value.keys())
+        return list(changed_ptypes)
+
+    def deploy(self, ptypes=None, force_deploy=False):
+        if not self.build:
+            raise DryccException("there are no builds available for deploy")
+        ptypes = set(ptypes).intersection(self.ptypes) if ptypes else self.ptypes
+        if not ptypes:
+            raise DryccException(
+                f'skip deploy, ptypes are not within the optional range: {self.ptypes}')
+        # change deployed_ptypes lock
+        msg = 'there is an executing pipeline, please wait or force deploy'
+        lock = self.app.lock()
         try:
-            envs = self.config.values
-            creds = self.get_registry_auth()
-
-            if self.build.type == "buildpack":
-                self.app.log('buildpack type detected. Defaulting to $PORT 5000')
-                return 5000
-
-            # application has registry auth - $PORT is required
-            if (creds is not None) or (settings.REGISTRY_LOCATION != 'on-cluster'):
-                if envs.get('PORT', None) is None:
-                    if not self.app.appsettings_set.latest().routable:
-                        return None
-                    raise DryccException(
-                        'PORT needs to be set in the application config '
-                        'when using a private registry'
-                    )
-
-                # User provided PORT
-                return int(envs.get('PORT'))
-
-            # If the user provides PORT
-            return int(envs.get('PORT', 5000))
-
-        except Exception as e:
-            raise DryccException(str(e)) from e
+            if lock.acquire() or force_deploy:
+                deployed_ptypes = list(set(self.deployed_ptypes).union(ptypes))
+                if deployed_ptypes:
+                    type(self).objects.filter(pk=self.pk).update(deployed_ptypes=deployed_ptypes)
+            else:
+                raise DryccException(msg)
+        finally:
+            lock.release()
+        # diff ptypes
+        if not force_deploy and self.app.appsettings_set.latest().autodeploy:
+            ptypes = self.diff_ptypes(ptypes)
+        # deploy lock
+        lock = DeployLock(self.app.pk)
+        if not lock.acquire(ptypes, force=force_deploy):
+            raise DryccException(f"{msg}: {lock.locked(ptypes)}")
+        run_pipeline.delay(self, ptypes, force_deploy)
+
+    @classmethod
+    def latest(cls, app, state=None, exclude_pk=None):
+        releases = app.release_set
+        if exclude_pk:
+            releases = releases.exclude(pk=exclude_pk)
+        q = Q(failed=False, state="succeed") if state else Q(failed=False)
+        try:
+            app_settings = app.appsettings_set.latest()
+            if app_settings.autorollback is False:
+                q = Q()
+        except AppSettings.DoesNotExist:
+            pass
+        try:
+            # Get the Release previous to this one
+            release = releases.filter(q).latest()
+        except Release.DoesNotExist:
+            release = None
+        return release
 
-    def get_registry_auth(self):
-        """
-        Gather login information for private registry if needed
-        """
-        auth = None
-        registry = self.config.registry
-        if registry.get('username', None):
-            auth = {
-                'username': registry.get('username', None),
-                'password': registry.get('password', None),
-                'email': self.owner.email
-            }
-
-        return auth
-
-    def previous(self):
+    def previous(self, state="succeed"):
         """
         Return the previous Release to this one.
-
         :return: the previous :class:`Release`, or None
         """
-        releases = self.app.release_set
-        if self.pk:
-            releases = releases.exclude(pk=self.pk)
+        return self.latest(self.app, state, self.pk)
 
-        try:
-            # Get the Release previous to this one
-            prev_release = releases.filter(failed=False).latest()
-        except Release.DoesNotExist:
-            prev_release = None
-        return prev_release
-
-    def rollback(self, user, version=None):
+    def rollback(self, user, ptypes=None, version=None):
         try:
             # if no version is provided then grab version from object
             version = (self.version - 1) if version is None else int(version)
@@ -131,37 +241,37 @@ def rollback(self, user, version=None):
                 raise DryccException('version cannot be below 0')
             elif version == 1:
                 raise DryccException('Cannot roll back to initial release.')
-
             prev = self.app.release_set.get(version=version)
             if prev.failed:
                 raise DryccException('Cannot roll back to failed release.')
-            app_settings = self.app.appsettings_set.latest()
             latest_version = self.app.release_set.latest().version
             new_release = self.new(
                 user,
                 build=prev.build,
                 config=prev.config,
                 summary="{} rolled back to v{}".format(user, version),
-                canary=len(app_settings.canaries) > 0,
             )
-
             if self.build is not None:
-                self.app.deploy(new_release, force_deploy=True)
+                new_release.deploy(ptypes, force_deploy=True)
             return new_release
         except Exception as e:
             # check if the exception is during create or publish
             if ('new_release' not in locals() and 'latest_version' in locals() and
                     self.app.release_set.latest().version == latest_version+1):
                 new_release = self.app.release_set.latest()
-            if 'new_release' in locals():
+                new_release.state = "crashed"
                 new_release.failed = True
-                new_release.summary = "{} performed roll back to a release that failed".format(self.owner)  # noqa
+                if new_release.summary:
+                    new_release.summary += " "
+                new_release.summary += "{} performed roll back to a release that failed".format(
+                    user)
                 # Get the exception that has occured
                 new_release.exception = "error: {}".format(str(e))
-                new_release.save()
+                # avoid overwriting other fields
+                new_release.save(update_fields=["state", "failed", "summary", "exception"])
             raise DryccException(str(e)) from e
 
-    def cleanup_old(self):  # noqa
+    def clean(self, ptypes=None):
         """
         Cleanup any old resources from Kubernetes
 
@@ -169,63 +279,68 @@ def cleanup_old(self):  # noqa
         Secrets no longer tied to any ReplicaSet
         Stray pods no longer relevant to the latest release
         """
-        latest_version = 'v{}'.format(self.version)
-        self.app.log(
-            'Cleaning up RSs for releases older than {} (latest)'.format(latest_version),
+        self.log(
+            'Cleaning up RSs for releases older than {} (latest)'.format(self.version_name),
             level=logging.DEBUG
         )
-
-        # Cleanup controllers
+        # base labels
         labels = {'heritage': 'drycc'}
+        if ptypes is not None:
+            labels['type__in'] = ptypes
+        # Cleanup controllers
         replica_sets_removal = []
-        replica_sets = self._scheduler.rs.get(self.app.id, labels=labels).json()['items']
+        replica_sets = self.scheduler.rs.get(self.app.id, labels=labels).json()['items']
         if not replica_sets:
             replica_sets = []
         for replica_set in replica_sets:
-            current_version = replica_set['metadata']['labels']['version']
-            # skip the latest release
-            if current_version == latest_version:
+            current_version_name = replica_set['metadata']['labels']['version']
+            # skip the later release
+            current_version = int(current_version_name[1:])
+            if current_version >= self.version:
                 continue
 
             # aggregate versions together to removal all at once
-            if current_version not in replica_sets_removal:
-                replica_sets_removal.append(current_version)
+            if current_version_name not in replica_sets_removal:
+                replica_sets_removal.append(current_version_name)
 
         if replica_sets_removal:
-            self.app.log(
-                'Found the following versions to cleanup: {}'.format(', '.join(replica_sets_removal)),  # noqa
+            self.log(
+                'Found the following versions to cleanup: {}'.format(
+                    ', '.join(replica_sets_removal)),
                 level=logging.DEBUG
             )
-
         # this is RC related
-        for version in replica_sets_removal:
-            self._delete_release_in_scheduler(self.app.id, version)
-
+        for version_name in replica_sets_removal:
+            self._delete_release_in_scheduler(self.app.id, ptypes, version_name)
         # handle Deployments specific cleanups
-        self._cleanup_deployment_secrets_and_configs(self.app.id)
-
+        self._cleanup_deployment_secrets_and_configs(self.app.id, ptypes)
         # Remove stray pods
+        self._cleanup_stray_pods(self.app.id, ptypes, self.version)
+
+    def _cleanup_stray_pods(self, namespace, ptypes, latest_version):
         labels = {'heritage': 'drycc'}
-        pods = self._scheduler.pod.get(self.app.id, labels=labels).json()['items']
+        if ptypes is not None:
+            labels['type__in'] = ptypes
+        pods = self.scheduler.pod.get(namespace, labels=labels).json()['items']
         if not pods:
             pods = []
         for pod in pods:
-            if self._scheduler.pod.deleted(pod):
+            if self.scheduler.pod.deleted(pod):
                 continue
 
-            current_version = pod['metadata']['labels']['version']
-            # skip the latest release
-            if current_version == latest_version:
+            current_version = int(pod['metadata']['labels']['version'][1:])
+            # skip the later release
+            if current_version >= latest_version:
                 continue
 
             try:
-                self._scheduler.pod.delete(self.app.id, pod['metadata']['name'])
+                self.scheduler.pod.delete(namespace, pod['metadata']['name'])
             except KubeHTTPException as e:
                 # Sometimes k8s will manage to remove the pod from under us
                 if e.response.status_code == 404:
                     continue
 
-    def _cleanup_deployment_secrets_and_configs(self, namespace):
+    def _cleanup_deployment_secrets_and_configs(self, namespace, ptypes=None):
         """
         Clean up any environment secrets (and in the future ConfigMaps) that
         are tied to a release Deployments no longer track
@@ -236,36 +351,41 @@ def _cleanup_deployment_secrets_and_configs(self, namespace):
         """
 
         # Find all ReplicaSets
-        versions = []
+        version_names = [self.version_name, ]
         labels = {'heritage': 'drycc', 'app': namespace}
-        replicasets = self._scheduler.rs.get(namespace, labels=labels).json()['items']
+        replicasets = self.scheduler.rs.get(namespace, labels=labels).json()['items']
         if not replicasets:
             replicasets = []
         for replicaset in replicasets:
+            replica_version = int(replicaset['metadata']['labels']['version'][1:])
+            # only remove the older releases secret
             if (
                 'version' not in replicaset['metadata']['labels'] or
-                replicaset['metadata']['labels']['version'] in versions
+                replica_version < self.version or
+                replicaset['metadata']['labels']['version'] in version_names
             ):
                 continue
 
-            versions.append(replicaset['metadata']['labels']['version'])
+            version_names.append(replicaset['metadata']['labels']['version'])
 
         # find all env secrets not owned by any replicaset
         labels = {
             'heritage': 'drycc',
             'app': namespace,
-            'type': 'env',
+            'class': 'env',
             # http://kubernetes.io/docs/user-guide/labels/#set-based-requirement
-            'version__notin': versions
+            'version__notin': version_names
         }
-        self.app.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
-        secrets = self._scheduler.secret.get(namespace, labels=labels).json()['items']
+        if ptypes is not None:
+            labels['type__in'] = ptypes
+        self.log('Cleaning up orphaned env var secrets for application {}'.format(namespace), level=logging.DEBUG)  # noqa
+        secrets = self.scheduler.secret.get(namespace, labels=labels).json()['items']
         if not secrets:
             secrets = []
         for secret in secrets:
-            self._scheduler.secret.delete(namespace, secret['metadata']['name'])
+            self.scheduler.secret.delete(namespace, secret['metadata']['name'])
 
-    def _delete_release_in_scheduler(self, namespace, version):
+    def _delete_release_in_scheduler(self, namespace, ptypes, version_name):
         """
         Deletes a specific release in k8s based on ReplicationController
 
@@ -275,20 +395,26 @@ def _delete_release_in_scheduler(self, namespace, version):
         labels = {
             'heritage': 'drycc',
             'app': namespace,
-            'version': version
+            'version': version_name
         }
-
-        # see if the app config has deploy timeout preference, otherwise use global
-        timeout = self.config.values.get('DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)
-
-        replica_sets = self._scheduler.rs.get(namespace, labels=labels).json()['items']
+        if ptypes is not None:
+            labels['type__in'] = ptypes
+        replica_sets = self.scheduler.rs.get(namespace, labels=labels).json()['items']
         if not replica_sets:
             replica_sets = []
         for replica_set in replica_sets:
+            # see if the app config has deploy timeout preference, otherwise use global
+            ptype = replica_set['metadata'].get("labels", {}).get("type", PTYPE_WEB)
+            timeout = self.config.envs(ptype).get(
+                'DRYCC_DEPLOY_TIMEOUT', settings.DRYCC_DEPLOY_TIMEOUT)
             # Deployment takes care of this in the API, RS does not
             # Have the RS scale down pods and delete itself
-            self._scheduler.rs.scale(namespace, replica_set['metadata']['name'], 0, timeout)
-            self._scheduler.rs.delete(namespace, replica_set['metadata']['name'])
+            try:
+                self.scheduler.rs.scale(namespace, replica_set['metadata']['name'], 0, timeout)
+                self.scheduler.rs.delete(namespace, replica_set['metadata']['name'])
+            except KubeHTTPException as e:
+                if e.response.status_code != 404:
+                    raise
 
     def save(self, *args, **kwargs):  # noqa
         if not self.summary:
@@ -299,145 +425,28 @@ def save(self, *args, **kwargs):  # noqa
             old_config = prev_release.config if prev_release else None
             # if the build changed, log it and who pushed it
             if self.version == 1:
-                self.summary += "{} created initial release".format(self.app.owner)
+                self.summary += "{} created initial release".format(self.app.workspace.name)
             elif self.build != old_build:
                 if self.build.sha:
-                    self.summary += "{} deployed {}".format(self.build.owner, self.build.sha[:7])
+                    self.summary += "{} deployed {}".format(
+                        self.app.workspace.name, self.build.sha[:7])
                 else:
-                    self.summary += "{} deployed {}".format(self.build.owner, self.build.image)
-
-            # if the config data changed, log the dict diff
-            if self.config != old_config:
-                # if env vars change, log the dict diff
-                dict1 = self.config.values
-                dict2 = old_config.values if old_config else {}
-                diff = dict_diff(dict1, dict2)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the limits changed (memory or cpu), log the dict diff
-                changes = []
-                old_mem = old_config.memory if old_config else {}
-                diff = dict_diff(self.config.memory, old_mem)
-                if diff.get('added') or diff.get('changed') or diff.get('deleted'):
-                    changes.append('memory')
-                old_cpu = old_config.cpu if old_config else {}
-                diff = dict_diff(self.config.cpu, old_cpu)
-                if diff.get('added') or diff.get('changed') or diff.get('deleted'):
-                    changes.append('cpu')
-                if changes:
-                    changes = 'changed limits for '+', '.join(changes)
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the lifecycle_post_start hooks changed, log the dict diff
-                changes = []
-                old_lifecycle_post_start = old_config.lifecycle_post_start if old_config else {}
-                diff = dict_diff(self.config.lifecycle_post_start, old_lifecycle_post_start)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added lifecycle_post_start  ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed lifecycle_post_start ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted lifecycle_post_start ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the lifecycle_pre_stop hooks changed, log the dict diff
-                changes = []
-                old_lifecycle_pre_stop = old_config.lifecycle_pre_stop if old_config else {}
-                diff = dict_diff(self.config.lifecycle_pre_stop, old_lifecycle_pre_stop)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added lifecycle_pre_stop  ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed lifecycle_pre_stop ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted lifecycle_pre_stop ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-
-                # if the timeouts changed, log the dict diff
-                changes = []
-                old_timeout = old_config.termination_grace_period if old_config else {}
-                diff = dict_diff(self.config.termination_grace_period, old_timeout)
-                if diff.get('added') or diff.get('changed') or diff.get('deleted'):
-                    changes.append('termination_grace_period')
-                if changes:
-                    changes = 'changed timeouts for '+', '.join(changes)
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the tags changed, log the dict diff
-                changes = []
-                old_tags = old_config.tags if old_config else {}
-                diff = dict_diff(self.config.tags, old_tags)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added tag ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed tag ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted tag ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the registry information changed, log the dict diff
-                changes = []
-                old_registry = old_config.registry if old_config else {}
-                diff = dict_diff(self.config.registry, old_registry)
-                # try to be as succinct as possible
-                added = ', '.join(k for k in diff.get('added', {}))
-                added = 'added registry info ' + added if added else ''
-                changed = ', '.join(k for k in diff.get('changed', {}))
-                changed = 'changed registry info ' + changed if changed else ''
-                deleted = ', '.join(k for k in diff.get('deleted', {}))
-                deleted = 'deleted registry info ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
-                # if the healthcheck information changed, log the dict diff
-                changes = []
-                old_healthcheck = old_config.healthcheck if old_config else {}
-                diff = dict_diff(self.config.healthcheck, old_healthcheck)
-                # try to be as succinct as possible
-                added = ', '.join(list(map(lambda x: 'default' if x == '' else x, [k for k in diff.get('added', {})])))  # noqa
-                added = 'added healthcheck info for proc type ' + added if added else ''
-                changed = ', '.join(list(map(lambda x: 'default' if x == '' else x, [k for k in diff.get('changed', {})])))  # noqa
-                changed = 'changed healthcheck info for proc type ' + changed if changed else ''
-                deleted = ', '.join(list(map(lambda x: 'default' if x == '' else x, [k for k in diff.get('deleted', {})])))  # noqa
-                deleted = 'deleted healthcheck info for proc type ' + deleted if deleted else ''
-                changes = ', '.join(i for i in (added, changed, deleted) if i)
-                if changes:
-                    if self.summary:
-                        self.summary += ' and '
-                    self.summary += "{} {}".format(self.config.owner, changes)
-
+                    self.summary += "{} deployed {}".format(
+                        self.app.workspace.name, self.build.image)
+            elif self.config != old_config:
+                for field, diff in self.config.diff(old_config).items():
+                    diff_list = []
+                    for diff_type, values in diff.items():
+                        diff_list.append(f'{diff_type} {field} {", ".join(values)}')
+                    if diff_list:
+                        changes = ', '.join(diff_list)
+                        self.summary += "{} {}".format(self.app.workspace.name, changes)
             if not self.summary:
                 if self.version == 1:
-                    self.summary = "{} created the initial release".format(self.owner)
+                    self.summary = "{} created the initial release".format(self.app.workspace.name)
                 else:
                     # There were no changes to this release
-                    raise AlreadyExists("{} changed nothing - release stopped".format(self.owner))
-
+                    raise AlreadyExists(
+                        "{} changed nothing - release stopped".format(self.app.workspace.name)
+                    )
         super(Release, self).save(*args, **kwargs)
diff --git a/rootfs/api/models/resource.py b/rootfs/api/models/resource.py
index d247f19bf..63ef0b8c2 100644
--- a/rootfs/api/models/resource.py
+++ b/rootfs/api/models/resource.py
@@ -1,8 +1,10 @@
 import logging
+from functools import cmp_to_key
 from django.db import models, transaction
 from django.contrib.auth import get_user_model
 from api.exceptions import DryccException, AlreadyExists, ServiceUnavailable
-from api.utils import validate_label
+from api.tasks import send_app_log
+from api.utils import validate_label, get_scheduler
 from scheduler import KubeException
 from .base import UuidAuditedModel
 
@@ -11,7 +13,6 @@
 
 
 class Resource(UuidAuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     name = models.CharField(max_length=63, validators=[validate_label])
     plan = models.CharField(max_length=128)
@@ -37,38 +38,43 @@ def save(self, *args, **kwargs):
         return super(Resource, self).save(*args, **kwargs)
 
     @classmethod
-    @property
     def services(cls):
-        services = []
-        for serviceclass in cls._scheduler.svcat.get_serviceclasses().json()["items"]:
+        services, scheduler = [], get_scheduler()
+        for serviceclass in scheduler.svcat.get_serviceclasses().json()["items"]:
             services.append({
                 "id": serviceclass["spec"]["externalID"],
                 "name": serviceclass["spec"]["externalName"],
                 "updateable": serviceclass["spec"]["planUpdatable"],
             })
+        services.sort(key=lambda service: service["name"])
         return services
 
     @classmethod
     def plans(cls, serviceclass_name):
-        serviceclass_id = None
-        for service in cls.services:
+        serviceclass_id, scheduler = None, get_scheduler()
+        for service in cls.services():
             if service["name"] == serviceclass_name:
                 serviceclass_id = service["id"]
                 break
         plans = []
         if serviceclass_id is not None:
-            for serviceplan in cls._scheduler.svcat.get_serviceplans().json()["items"]:
+            for serviceplan in scheduler.svcat.get_serviceplans().json()["items"]:
                 if serviceplan["spec"]["clusterServiceClassRef"]["name"] == serviceclass_id:
                     plans.append({
                         "id": serviceplan["spec"]["externalID"],
                         "name": serviceplan["spec"]["externalName"],
                         "description": serviceplan["spec"]["description"],
                     })
+        plans.sort(key=cmp_to_key(
+            lambda p1, p2: len(p1["name"]) - len(p2["name"])
+            if len(p1["name"]) != len(p2["name"])
+            else (1 if p1["name"] > p2["name"] else -1)
+        ))
         return plans
 
     def attach(self, *args, **kwargs):
         try:
-            self._scheduler.svcat.get_instance(self.app.id, self.name)
+            self.scheduler.svcat.get_instance(self.app.id, self.name)
             err = "Resource {} already exists in this namespace".format(self.name)  # noqa
             self.log(err, logging.INFO)
             raise AlreadyExists(err)
@@ -81,7 +87,7 @@ def attach(self, *args, **kwargs):
                     "instance_plan": ":".join(instance[1:]),
                     "parameters": self.options,
                 }
-                self._scheduler.svcat.create_instance(
+                self.scheduler.svcat.create_instance(
                     self.app.id, self.name, **kwargs
                 )
             except KubeException as e:
@@ -102,10 +108,10 @@ def delete(self, *args, **kwargs):
 
     def detach(self, *args, **kwargs):
         try:
-            resp = self._scheduler.svcat.get_instance(
+            resp = self.scheduler.svcat.get_instance(
                 self.app.id, self.name, ignore_exception=True)
             if resp.status_code != 404:
-                self._scheduler.svcat.delete_instance(self.app.id, self.name)
+                self.scheduler.svcat.delete_instance(self.app.id, self.name)
         except KubeException as e:
             raise ServiceUnavailable("Could not delete resource {} for application {}".format(self.name, self.app_id)) from e  # noqa
 
@@ -118,7 +124,8 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
-        logger.log(level, "[{}]: {}".format(self.uuid, message))
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
 
     def bind(self, *args, **kwargs):
         if self.status != "Ready":
@@ -128,14 +135,14 @@ def bind(self, *args, **kwargs):
         self.binding = "Binding"
         self.save()
         try:
-            self._scheduler.svcat.get_binding(self.app.id, self.name)
+            self.scheduler.svcat.get_binding(self.app.id, self.name)
             err = "Resource {} is binding".format(self.name)
             self.log(err, logging.INFO)
             raise AlreadyExists(err)
         except KubeException as e:
             logger.info(e)
             try:
-                self._scheduler.svcat.create_binding(
+                self.scheduler.svcat.create_binding(
                     self.app.id, self.name, **kwargs)
             except KubeException as e:
                 msg = 'There was a problem binding the resource ' \
@@ -147,8 +154,8 @@ def unbind(self, *args, **kwargs):
             raise DryccException("the resource instance is not binding")
         try:
             # We raise an exception when a resource doesn't exist
-            self._scheduler.svcat.get_binding(self.app.id, self.name)
-            self._scheduler.svcat.delete_binding(self.app.id, self.name)
+            self.scheduler.svcat.get_binding(self.app.id, self.name)
+            self.scheduler.svcat.delete_binding(self.app.id, self.name)
             self.binding = None
             self.data = {}
             self.save()
@@ -157,7 +164,7 @@ def unbind(self, *args, **kwargs):
 
     def attach_update(self, *args, **kwargs):
         try:
-            data = self._scheduler.svcat.get_instance(
+            data = self.scheduler.svcat.get_instance(
                 self.app.id, self.name).json()
         except KubeException as e:
             logger.debug(e)
@@ -171,7 +178,7 @@ def attach_update(self, *args, **kwargs):
                 "parameters": self.options,
                 "external_id": data["spec"]["externalID"]
             }
-            self._scheduler.svcat.patch_instance(
+            self.scheduler.svcat.patch_instance(
                 self.app.id, self.name, version, **kwargs
             )
         except KubeException as e:
@@ -182,12 +189,13 @@ def attach_update(self, *args, **kwargs):
     @property
     def message(self):
         try:
-            resp = self._scheduler.svcat.get_instance(
+            resp = self.scheduler.svcat.get_instance(
                 self.app.id, self.name)
             if resp.status_code != 200:
                 message = ""
-            message = resp.json().get("status", {}).get("conditions", [{}])[-1].\
-                get("message", "")
+            conditions = resp.json().get("status", {}).get("conditions")
+            message = conditions[-1].get("message", "") \
+                if conditions and isinstance(conditions[-1], dict) else ""
             return message
         except KubeException as e:
             logger.info("retrieve instance info error: {}".format(e))
@@ -198,21 +206,25 @@ def retrieve(self, *args, **kwargs):
             self.save()
         return self.status == self.binding == "Ready"
 
-    def to_measurements(self, timestamp: float):
+    def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
-            "owner": self.owner_id,
-            "name": self.name,
-            "type": self.plan,
+            "workspace": self.app.workspace_id,
+            "name": self.plan,
+            "type": "resource",
             "unit": "number",
             "usage": 1,
-            "timestamp": int(timestamp)
+            "kwargs": {
+                "name": self.name,
+            },
+            "timestamp": int(timestamp),
+            "identifier": self.uuid.hex,
         }]
 
     def _retrieve_status(self):
         changed = False
         try:
-            response = self._scheduler.svcat.get_instance(
+            response = self.scheduler.svcat.get_instance(
                 self.app.id, self.name).json()
             status = response.get('status', {}).get('lastConditionState')
             options = response.get('spec', {}).get('parameters', {})
@@ -230,14 +242,14 @@ def _retrieve_binding(self):
         changed = False
         try:
             # We raise an exception when a resource doesn't exist
-            response = self._scheduler.svcat.get_binding(self.app.id, self.name).json()
+            response = self.scheduler.svcat.get_binding(self.app.id, self.name).json()
             binding = response.get('status', {}).get('lastConditionState')
             secret_name = response.get('spec', {}).get('secretName')
             if self.binding != binding:
                 self.binding = binding
                 changed = True
             if secret_name:
-                response = self._scheduler.secret.get(self.app.id, secret_name).json()
+                response = self.scheduler.secret.get(self.app.id, secret_name).json()
                 data = response.get('data', {})
                 if self.data != data:
                     self.data = data
diff --git a/rootfs/api/models/service.py b/rootfs/api/models/service.py
index d1b3d34e6..6b1c2711f 100644
--- a/rootfs/api/models/service.py
+++ b/rootfs/api/models/service.py
@@ -1,62 +1,116 @@
 import logging
-
+from functools import partial
 from django.db import models
-from django.conf import settings
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
+from api.utils import validate_json
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
-from .base import AuditedModel
+from .base import AuditedModel, PTYPE_MAX_LENGTH
+
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
+service_ports_schema = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "array",
+    "minItems": 1,
+    "items": {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "port": {"type": "integer"},
+            "protocol": {"type": "string"},
+            "targetPort": {"type": "integer"},
+        },
+        "required": ["name", "port", "protocol", "targetPort"],
+    }
+}
 
 
 class Service(AuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
-    ports = models.JSONField(default=list)
-    canary = models.BooleanField(default=False)
-    procfile_type = models.TextField()
+    ports = models.JSONField(
+        default=list, validators=[partial(validate_json, schema=service_ports_schema)])
+    ptype = models.CharField(max_length=PTYPE_MAX_LENGTH)
 
     class Meta:
         get_latest_by = 'created'
-        unique_together = (('app', 'procfile_type'), )
+        unique_together = (('app', 'ptype'), )
         ordering = ['-created']
 
     def __str__(self):
-        return self._svc_name(False)
+        return self.name
+
+    @property
+    def name(self):
+        if self.ptype == 'web':
+            svc_name = self.app.id
+        else:
+            svc_name = "{}-{}".format(self.app.id, self.ptype)
+        return svc_name
 
     @property
     def domain(self):
-        return "{}.{}.svc.{}".format(
-            self._svc_name(False), self._namespace(), settings.KUBERNETES_CLUSTER_DOMAIN
-        )
+        return "{}.{}.svc".format(self.name, self.namespace)
+
+    @property
+    def namespace(self):
+        return self.app.id
+
+    @classmethod
+    def get(cls, app, name):
+        for service in cls.objects.filter(app=app):
+            if service.name == name:
+                return service
+        raise cls.DoesNotExist()
 
     def as_dict(self):
         return {
+            "name": self.name,
             "domain": self.domain,
             "ports": self.ports,
-            "canary": self.canary,
-            "procfile_type": self.procfile_type,
+            "ptype": self.ptype,
         }
 
     def port_name(self, port, protocol):
-        return "-".join([self.app.id, self.procfile_type, protocol, str(port)]).lower()
+        return "-".join([protocol, str(port)]).lower()
 
-    def add_port(self, port, protocol, target_port):
-        self.ports.append({
-            "name": self.port_name(port, protocol),
-            "port": port,
-            "protocol": protocol,
-            "targetPort": target_port,
-        })
+    def get_port(self, port, protocol):
+        for item in self.ports:
+            if item["port"] == port and item["protocol"] == protocol:
+                return item
+        return None
 
-    def refresh_k8s_svc(self):
-        if self.canary:
-            self._refresh_k8s_svc(self._svc_name(False))
-        else:
-            self._delete_k8s_svc(self._svc_name(True))
-        self._refresh_k8s_svc(self._svc_name(self.canary))
+    def add_port(self, port, protocol, target_port):
+        if self.get_port(port, protocol) is None:
+            self.ports.append({
+                "name": self.port_name(port, protocol),
+                "port": port,
+                "protocol": protocol,
+                "targetPort": target_port,
+            })
+            return True
+        return False
+
+    def update_port(self, port, protocol, target_port):
+        item = self.get_port(port, protocol)
+        if not item or item["targetPort"] != target_port:
+            if item and item["targetPort"] != target_port:
+                self.remove_port(port, protocol)
+            self.add_port(port, protocol, target_port)
+            return True
+        return False
+
+    def remove_port(self, port, protocol):
+        ports = []
+        for item in self.ports:
+            if item["port"] != port or item["protocol"] != protocol:
+                ports.append(item)
+        if len(self.ports) > len(ports):
+            self.ports = ports
+            return True
+        return False
 
     def save(self, *args, **kwargs):
         service = super(Service, self).save(*args, **kwargs)
@@ -64,9 +118,7 @@ def save(self, *args, **kwargs):
         return service
 
     def delete(self, *args, **kwargs):
-        if self.canary:
-            self._delete_k8s_svc(self._svc_name(False))
-        self._delete_k8s_svc(self._svc_name(self.canary))
+        self._delete_k8s_svc(self.name)
         # Delete from DB
         return super(Service, self).delete(*args, **kwargs)
 
@@ -79,45 +131,27 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
-        logger.log(level, "[{}]: {}".format(self.id, message))
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
 
-    def _namespace(self):
-        return self.app.id
-
-    def _svc_name(self, canary):
-        if self.procfile_type == 'web':
-            svc_name = self.app.id
-        else:
-            svc_name = "{}-{}".format(self.app.id, self.procfile_type)
-        if canary:
-            svc_name = "%s-canary" % svc_name
-        return svc_name
-
-    def _refresh_k8s_svc(self, svc_name):
-        namespace = self._namespace()
-        self.log('creating service: {}'.format(svc_name), level=logging.DEBUG)
+    def refresh_k8s_svc(self):
+        self.log('creating service: {}'.format(self.name), level=logging.DEBUG)
         try:
             try:
-                data = self._scheduler.svc.get(namespace, svc_name).json()
-                self._scheduler.svc.patch(namespace, svc_name, **{
+                data = self.scheduler.svc.get(self.namespace, self.name).json()
+                self.scheduler.svc.patch(self.namespace, self.name, **{
                     "ports": self.ports,
                     "version": data["metadata"]["resourceVersion"],
-                    "procfile_type": self.procfile_type,
+                    "ptype": self.ptype,
                 })
             except KubeException:
-                self._scheduler.svc.create(namespace, svc_name, **{
+                self.scheduler.svc.create(self.namespace, self.name, **{
                     "ports": self.ports,
-                    "procfile_type": self.procfile_type,
+                    "ptype": self.ptype,
                 })
         except KubeException as e:
             raise ServiceUnavailable('Kubernetes service could not be created') from e
 
     def _delete_k8s_svc(self, svc_name):
         self.log('deleting Service: {}'.format(svc_name), level=logging.DEBUG)
-        try:
-            self._scheduler.svc.delete(self._namespace(), svc_name)
-        except KubeException:
-            # swallow exception
-            # raise ServiceUnavailable('Kubernetes service could not be deleted') from e
-            self.log('Kubernetes service cannot be deleted: {}'.format(svc_name),
-                     level=logging.ERROR)
+        self.scheduler.svc.delete(self.namespace, svc_name, ignore_exception=True)
diff --git a/rootfs/api/models/tls.py b/rootfs/api/models/tls.py
index d091b722a..b6a5b9262 100644
--- a/rootfs/api/models/tls.py
+++ b/rootfs/api/models/tls.py
@@ -3,6 +3,7 @@
 from django.db import models
 from django.db import transaction
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.exceptions import AlreadyExists
 from api.exceptions import ServiceUnavailable
 from scheduler import KubeException
@@ -23,16 +24,101 @@ def default_issuer():
 
 
 class TLS(UuidAuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     issuer = models.JSONField(default=default_issuer)
     https_enforced = models.BooleanField(null=True)
     certs_auto_enabled = models.BooleanField(null=True)
 
-    class Meta:
-        get_latest_by = 'created'
-        unique_together = (('app', 'uuid'))
-        ordering = ['-created']
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this application.
+
+        This prefixes log messages with an application "tag" that the customized
+        drycc-logspout will be on the lookout for.  When it's seen, the message-- usually
+        an application event of some sort like releasing or scaling, will be considered
+        as "belonging" to the application instead of the controller and will be handled
+        accordingly.
+        """
+        send_app_log.delay(self.app.id, message, level)
+        logger.log(level, "[{}]: {}".format(self.app.id, message))
+
+    @property
+    def events(self):
+        def to_result(name, kind, condition):
+            return {
+                "name": name,
+                "kind": kind,
+                "time": condition["lastTransitionTime"],
+                "type": condition["type"],
+                "status": condition["status"],
+                "message": condition["message"],
+            }
+
+        results = []
+        name = namespace = self.app.id
+        response = self.scheduler.issuer.get(namespace, name, ignore_exception=True)
+        if response.status_code == 200:
+            for condition in response.json()["status"]["conditions"]:
+                results.append(to_result(name, "Issuer", condition))
+        name = f"{self.app.id}-auto-tls"
+        response = self.scheduler.certificate.get(namespace, name, ignore_exception=True)
+        if response.status_code == 200:
+            for condition in response.json()["status"]["conditions"]:
+                results.append(to_result(name, "Certificate", condition))
+        response = self.scheduler.certificaterequest.get(namespace, ignore_exception=True)
+        if response.status_code == 200:
+            for item in response.json()["items"]:
+                for condition in item["status"]["conditions"]:
+                    results.append(to_result(
+                        item["metadata"]["name"], "CertificateRequest", condition))
+        return results
+
+    def refresh_issuer_to_k8s(self):
+        name = namespace = self.app.id
+        try:
+            if self.issuer["key_id"] and self.issuer["key_secret"]:
+                self._refresh_secret_to_k8s()
+            data = copy.deepcopy(self.issuer)
+            data["parent_refs"] = [
+                {
+                    "group": "gateway.networking.k8s.io",
+                    "kind": "Gateway",
+                    "name": gateway.name,
+                }
+                for gateway in self.app.gateway_set.all()
+            ]
+            try:
+                version = self.scheduler.issuer.get(
+                    namespace, name, ignore_exception=False).json()["metadata"]["resourceVersion"]
+                data.update({"version": version})
+                self.scheduler.issuer.put(namespace, name, **data)
+            except KubeException:
+                self.scheduler.issuer.create(namespace, name, **data)
+        except KubeException as e:
+            raise ServiceUnavailable('Kubernetes issuer could not be created') from e
+
+    def refresh_certificate_to_k8s(self):
+        namespace, name = self.app.id, f"{self.app.id}-auto-tls"
+        if self.certs_auto_enabled:
+            hosts = [domain.domain for domain in self.app.domain_set.all()]
+            if len(hosts) > 0:
+                response = self.scheduler.certificate.get(namespace, name)
+                if response.status_code == 200:
+                    data = response.json()
+                    version = data["metadata"]["resourceVersion"]
+                    self.scheduler.certificate.put(namespace, name, hosts, version)
+                else:
+                    self.log(
+                        "certificate {} does not exist".format(namespace), level=logging.INFO)
+                    self.scheduler.certificate.create(namespace, name, hosts)
+            else:
+                self.log("skip creating certificate, no domain name set", logging.WARNING)
+        else:
+            self.scheduler.certificate.delete(namespace, name, ignore_exception=True)
+
+    @transaction.atomic
+    def save(self, *args, **kwargs):
+        self._check_previous_tls_settings()
+        super(TLS, self).save(*args, **kwargs)
 
     def __str__(self):
         return "{}-{}".format(self.app.id, str(self.uuid)[:7])
@@ -47,12 +133,12 @@ def _check_previous_tls_settings(self):
             if self.https_enforced is not None:
                 if previous_tls_settings.https_enforced == self.https_enforced:
                     raise AlreadyExists(
-                        "{} changed nothing".format(self.owner))
+                        "{} changed nothing".format(self.app.workspace.name))
                 self.certs_auto_enabled = previous_tls_settings.certs_auto_enabled
             elif self.certs_auto_enabled is not None:
                 if previous_tls_settings.certs_auto_enabled == self.certs_auto_enabled:
                     raise AlreadyExists(
-                        "{} changed nothing".format(self.owner))
+                        "{} changed nothing".format(self.app.workspace.name))
                 self.https_enforced = previous_tls_settings.https_enforced
             previous_tls_settings.delete()
         except TLS.DoesNotExist:
@@ -62,58 +148,19 @@ def _refresh_secret_to_k8s(self):
         secret_name = f"{self.app.id}-acme-external-account-binding-secret"
         try:
             try:
-                data = self._scheduler.secret.get(self.app.id, secret_name).json()
-                self._scheduler.secret.patch(self.app.id, secret_name, {
+                data = self.scheduler.secret.get(self.app.id, secret_name).json()
+                self.scheduler.secret.patch(self.app.id, secret_name, {
                     "secret": self.issuer["key_secret"],
                     "version": data["metadata"]["resourceVersion"],
                 })
             except KubeException:
-                self._scheduler.secret.create(self.app.id, secret_name, {
+                self.scheduler.secret.create(self.app.id, secret_name, {
                     "secret": self.issuer["key_secret"],
                 })
         except KubeException as e:
             raise ServiceUnavailable('Kubernetes secret could not be created') from e
 
-    def refresh_issuer_to_k8s(self):
-        name = namespace = self.app.id
-        try:
-            if self.issuer["key_id"] and self.issuer["key_secret"]:
-                self._refresh_secret_to_k8s()
-            data = copy.copy(self.issuer)
-            data["parent_refs"] = [
-                {
-                    "group": "gateway.networking.k8s.io",
-                    "kind": "Gateway",
-                    "name": gateway.name,
-                }
-                for gateway in self.app.gateway_set.all()
-            ]
-            try:
-                version = self._scheduler.issuer.get(
-                    namespace, name, ignore_exception=False).json()["metadata"]["resourceVersion"]
-                data.update({"version": version})
-                self._scheduler.issuer.put(namespace, name, **data)
-            except KubeException:
-                self._scheduler.issuer.create(namespace, name, **data)
-        except KubeException as e:
-            raise ServiceUnavailable('Kubernetes issuer could not be created') from e
-
-    def refresh_certificate_to_k8s(self):
-        namespace = name = self.app.id
-        if self.certs_auto_enabled:
-            hosts = [domain.domain for domain in self.app.domain_set.all()]
-            try:
-                data = self._scheduler.certificate.get(namespace, name).json()
-                version = data["metadata"]["resourceVersion"]
-                self._scheduler.certificate.put(namespace, name, hosts, version)
-            except KubeException:
-                logger.log(
-                    msg="certificate {} does not exist".format(namespace), level=logging.INFO)
-                self._scheduler.certificate.create(namespace, name, hosts)
-        else:
-            self._scheduler.certificate.delete(namespace, name, ignore_exception=True)
-
-    @transaction.atomic
-    def save(self, *args, **kwargs):
-        self._check_previous_tls_settings()
-        super(TLS, self).save(*args, **kwargs)
+    class Meta:
+        get_latest_by = 'created'
+        unique_together = (('app', 'uuid'))
+        ordering = ['-created']
diff --git a/rootfs/api/models/volume.py b/rootfs/api/models/volume.py
index 964eb0b3b..01a41b54c 100644
--- a/rootfs/api/models/volume.py
+++ b/rootfs/api/models/volume.py
@@ -1,9 +1,15 @@
 import logging
+import copy
+import json
+import uuid
+import hashlib
 from django.db import models, transaction
 from django.conf import settings
+from django.template import Template, Context
 from django.contrib.auth import get_user_model
+from api.tasks import send_app_log
 from api.utils import unit_to_bytes, validate_label
-from api.exceptions import DryccException, ServiceUnavailable, AlreadyExists
+from api.exceptions import DryccException, ServiceUnavailable
 from scheduler import KubeException
 from .base import UuidAuditedModel
 
@@ -12,46 +18,47 @@
 
 
 class Volume(UuidAuditedModel):
-    owner = models.ForeignKey(User, on_delete=models.PROTECT)
+    TYPE_CHOICES = (
+        ("csi", "container storage interface"),
+        ("nfs", "network file system"),
+        ("oss", "object storage service file"),
+    )
     app = models.ForeignKey('App', on_delete=models.CASCADE)
     name = models.CharField(max_length=63, validators=[validate_label])
-    size = models.CharField(max_length=128)
-    path = models.JSONField(default=dict, blank=True)
+    size = models.CharField(default='0G', max_length=128)
+    path = models.JSONField(default=dict)
+    type = models.CharField(default=TYPE_CHOICES[0][0], choices=TYPE_CHOICES)
+    parameters = models.JSONField(default=dict)
+
+    @property
+    def pv_name(self):
+        md5 = hashlib.md5()
+        md5.update(f"{self.app.id}-{self.type}-{self.name}".encode("utf-8"))
+        hexdigest = md5.hexdigest()
+        return "pvc-{}".format("-".join([
+            hexdigest[:8],
+            hexdigest[8:12],
+            hexdigest[12:16],
+            hexdigest[16:20],
+            hexdigest[20:]
+        ]))
 
     @transaction.atomic
     def save(self, *args, **kwargs):
         # Attach volume, updates k8s
-        if self.created == self.updated:
-            self._create_pvc()
+        self.save_to_k8s()
+        # check path
+        self.check_path()
         # Save to DB
         return super(Volume, self).save(*args, **kwargs)
 
     @transaction.atomic
     def delete(self, *args, **kwargs):
-        if self.path:
-            raise DryccException("the volume is not unmounted")
         # Deatch volume, updates k8s
-        self._delete_pvc()
+        self.delete_from_k8s()
         # Delete from DB
         return super(Volume, self).delete(*args, **kwargs)
 
-    @transaction.atomic
-    def expand(self, size):
-        if unit_to_bytes(size) < unit_to_bytes(self.size):
-            raise DryccException('Shrink volume is not supported.')
-        self.size = size
-        self.save()
-        try:
-            kwargs = {
-                "size": self._get_size(self.size),
-                "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
-            }
-            self._scheduler.pvc.patch(self.app.id, self.name, **kwargs)
-        except KubeException as e:
-            msg = 'There was a problem expand the volume ' \
-                    '{} for {}'.format(self.name, self.app_id)
-            raise ServiceUnavailable(msg) from e
-
     def log(self, message, level=logging.INFO):
         """Logs a message in the context of this service.
 
@@ -61,24 +68,54 @@ def log(self, message, level=logging.INFO):
         as "belonging" to the application instead of the controller and will be handled
         accordingly.
         """
+        send_app_log.delay(self.app.id, message, level)
         logger.log(level, "[{}]: {}".format(self.app.id, message))
 
-    def to_measurements(self, timestamp: float):
+    def to_usages(self, timestamp: float):
         return [{
             "app_id": str(self.app_id),
-            "owner": self.owner_id,
-            "name": self.name,
+            "workspace": self.app.workspace_id,
+            "name": self.type,
             "type": "volume",
             "unit": "bytes",
             "usage": unit_to_bytes(self.size),
+            "kwargs": {
+                "name": self.name,
+            },
             "timestamp": int(timestamp)
         }]
 
     def __str__(self):
         return self.name
 
+    def check_path(self, path=None):
+        other_volumes = self.app.volume_set.exclude(name=self.name)
+        type_paths = {}  # {'type1':[path1,path2], tyep2:[path3,path4]}
+        for _ in other_volumes:
+            for k, v in _.path.items():
+                if k not in type_paths:
+                    type_paths[k] = [v]
+                else:
+                    type_paths[k].append(v)
+        items = path.items() if path else self.path.items()
+        repeat_path = [v for k, v in items if v in type_paths.get(k, [])]
+        if repeat_path:
+            msg = "path {} is used by another volume".format(','.join(repeat_path))
+            self.log(msg, logging.ERROR)
+            raise DryccException(msg)
+
+    def save_to_k8s(self):
+        if self.type in settings.DRYCC_VOLUME_TEMPLATE:
+            self._save_pv()
+        self._save_pvc()
+
+    def delete_from_k8s(self):
+        if self.type in settings.DRYCC_VOLUME_TEMPLATE:
+            self._delete_pv()
+        self._delete_pvc()
+
     @staticmethod
-    def _get_size(size):
+    def _format_size(size):
         """ Format volume limit value """
         if size[-2:-1].isalpha() and size[-1].isalpha():
             size = size[:-1]
@@ -87,33 +124,75 @@ def _get_size(size):
             size = size.upper() + "i"
         return size
 
-    def _create_pvc(self):
+    def _save_pv(self):
+        kwds = copy.deepcopy(self.parameters.get(self.type, {}))
+        kwds.update({
+            "type": self.type,
+            "volume_claim_name": self.name,
+            "namespace": self.app.id,
+            "volume_handle": "%s" % uuid.uuid4(),
+        })
+        json_str = Template(settings.DRYCC_VOLUME_TEMPLATE).render(Context(kwds)).strip()
+        if not json_str:
+            return
+        kwargs = json.loads(json_str)
         try:
-            self._scheduler.pvc.get(self.app.id, self.name)
-            err = "Volume {} already exists in this namespace".format(self.name)  # noqa
-            self.log(err, logging.INFO)
-            raise AlreadyExists(err)
+            self.scheduler.pv.get(self.pv_name)
+            msg = "Volume {} already exists".format(self.pv_name)
+            self.log(msg, logging.INFO)
+            if "csi" in kwargs["spec"]:
+                del kwargs["spec"]["csi"]  # Fix: 422 unprocessable Entity
+            self.scheduler.pv.patch(self.pv_name, **kwargs)
         except KubeException as e:
             logger.info(e)
             try:
-                kwargs = {
-                    "size": self._get_size(self.size),
-                    "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
-                }
-                self._scheduler.pvc.create(self.app.id, self.name, **kwargs)
+                self.scheduler.pv.create(self.pv_name, **kwargs)
             except KubeException as e:
                 msg = 'There was a problem creating the volume ' \
+                      '{} for {}'.format(self.pv_name, self.app_id)
+                raise ServiceUnavailable(msg) from e
+
+    def _delete_pv(self):
+        try:
+            # We raise an exception when a volume doesn't exist
+            self.scheduler.pv.get(self.pv_name)
+            self.scheduler.pv.delete(self.pv_name)
+        except KubeException as e:
+            logger.exception(e)
+
+    def _save_pvc(self):
+        kwds = copy.deepcopy(self.parameters.get(self.type, {}))
+        kwds.update({
+            "type": self.type,
+            "size": self._format_size(self.size),
+            "volume_name": self.pv_name,
+            "storage_class": settings.DRYCC_APP_STORAGE_CLASS,
+        })
+        json_str = Template(settings.DRYCC_VOLUME_CLAIM_TEMPLATE).render(Context(kwds)).strip()
+        if not json_str:
+            raise DryccException('Volume claim template render error, context: {}'.format(kwds))
+        kwargs = json.loads(json_str)
+        try:
+            self.scheduler.pvc.get(self.app.id, self.name)
+            msg = "Volume claim {} already exists in this namespace".format(self.name)
+            self.log(msg, logging.INFO)
+            self.scheduler.pvc.patch(self.app.id, self.name, **kwargs)
+        except KubeException as e:
+            logger.info(e)
+            try:
+                self.scheduler.pvc.create(self.app.id, self.name, **kwargs)
+            except KubeException as e:
+                msg = 'There was a problem creating the volume claim ' \
                       '{} for {}'.format(self.name, self.app_id)
                 raise ServiceUnavailable(msg) from e
 
     def _delete_pvc(self):
         try:
             # We raise an exception when a volume doesn't exist
-            self._scheduler.pvc.get(self.app.id, self.name)
-            self._scheduler.pvc.delete(self.app.id, self.name)
+            self.scheduler.pvc.get(self.app.id, self.name)
+            self.scheduler.pvc.delete(self.app.id, self.name)
         except KubeException as e:
-            raise ServiceUnavailable("Could not delete volume {} for application \
-                {}".format(self.name, self.app_id)) from e  # noqa
+            logger.exception(e)
 
     class Meta:
         get_latest_by = 'created'
diff --git a/rootfs/api/models/workspace.py b/rootfs/api/models/workspace.py
new file mode 100644
index 000000000..7e5b6a689
--- /dev/null
+++ b/rootfs/api/models/workspace.py
@@ -0,0 +1,108 @@
+import re
+import logging
+from django.db import models
+from django.core.mail import send_mail
+from django.utils.translation import gettext_lazy as _
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from django.template.loader import render_to_string
+
+from rest_framework.exceptions import ValidationError
+from api.utils import validate_reserved_names, get_local_host
+
+User = get_user_model()
+logger = logging.getLogger(__name__)
+
+
+def validate_workspace_name(value):
+    """
+    Check that the value follows the kubernetes name constraints
+    """
+    match = re.match(r'^[0-9a-z]{5,}$', value)
+    if not match:
+        raise ValidationError("App name must start with an alphabetic character, cannot end with a"
+                              + " hyphen and can only contain a-z (lowercase), 0-9 and hyphens.")
+    validate_reserved_names(value)
+
+
+class Workspace(models.Model):
+    name = models.SlugField(
+        _("workspace name"),
+        max_length=150,
+        unique=True,
+        validators=[validate_workspace_name],
+    )
+    email = models.EmailField(_("email address"))
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+
+    def has_member(self, user, role=None):
+        kwargs = {'user': user, 'workspace': self}
+        if role:
+            kwargs['role'] = role
+        return WorkspaceMember.objects.filter(**kwargs).exists()
+
+    def __str__(self):
+        return self.name
+
+
+class WorkspaceMember(models.Model):
+    role_choices = [
+        ('admin', 'Admin'),
+        ('member', 'Member'),
+        ('viewer', 'Viewer'),
+    ]
+
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    role = models.CharField(max_length=50, choices=role_choices)
+    alerts = models.BooleanField(default=True)
+    created = models.DateTimeField(auto_now_add=True)
+    updated = models.DateTimeField(auto_now=True)
+    workspace = models.ForeignKey(Workspace, on_delete=models.CASCADE)
+
+    def __str__(self):
+        return f"{self.user.username} - {self.workspace.name} ({self.role})"
+
+    class Meta:
+        unique_together = ('user', 'workspace')
+
+
+class WorkspaceInvitation(models.Model):
+    email = models.EmailField(_("email address"))
+    token = models.CharField(max_length=128, unique=True)
+    inviter = models.ForeignKey(User, on_delete=models.CASCADE)
+    created = models.DateTimeField(auto_now_add=True)
+    accepted = models.BooleanField(_("accepted"), default=False)
+    workspace = models.ForeignKey(Workspace, on_delete=models.CASCADE)
+
+    class Meta:
+        unique_together = ('email', 'workspace')
+
+    def accept(self):
+        if not self.accepted:
+            self.accepted = True
+            self.save()
+        user = User.objects.filter(email=self.email).first()
+        if not user:
+            return
+        WorkspaceMember.objects.get_or_create(
+            user=user, workspace=self.workspace, defaults={'role': 'member'})
+
+    def send_email(self, request):
+        cache_key = f"invitation:email:{self.email}"
+        cache.add(cache_key, 0, timeout=settings.DRYCC_INVITATION_EMAIL_TIMEOUT)
+        count = cache.incr(cache_key)
+        if count > settings.DRYCC_INVITATION_EMAIL_LIMIT:
+            raise ValidationError("Too many invitation emails, please try again later")
+        domain = get_local_host(request)
+        mail_subject = f'We Invite You to Join the {self.workspace.name} Workspace.'
+        message = render_to_string(
+            'workspace/workspace_invitation.html',
+            {'domain': domain, 'invitation': self}
+        )
+        send_mail(mail_subject, message, None, [self.email], fail_silently=True)
+
+    def __str__(self):
+        return f"Invitation for {self.email} to join {self.workspace.name}"
diff --git a/rootfs/api/monitor.py b/rootfs/api/monitor.py
index 145f85d79..df8a1f79e 100644
--- a/rootfs/api/monitor.py
+++ b/rootfs/api/monitor.py
@@ -1,176 +1,62 @@
-import requests
-from typing import Iterator, Dict
-from contextlib import closing
-from urllib.parse import urljoin
-from django.db import connections
+import time
+import aiohttp
+from typing import Iterator, AsyncGenerator
 from django.conf import settings
-
-
-query_network_flow_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  last(rx_bytes, time) - first(rx_bytes, time) as rx_bytes,
-  last(tx_bytes, time) - first(tx_bytes, time) as tx_bytes
-FROM kubernetes_pod_network
-LEFT OUTER JOIN kubernetes_pod_network_tag
-ON kubernetes_pod_network.tag_id = kubernetes_pod_network_tag.tag_id
-WHERE
-  namespace in ({namespace_range})
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP by namespace, pod_name
-"""
-
-
-query_container_count_sql_tpl = """
-SELECT
-  count(1)
-FROM (
-  SELECT
-    namespace,
-    pod_name,
-    container_name
-  FROM kubernetes_pod_container
-  LEFT OUTER JOIN kubernetes_pod_container_tag
-  ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id
-  WHERE
-    namespace='{namespace}'
-    AND container_name='{container_name}'
-    AND time > to_timestamp({start})
-    AND time < to_timestamp({stop})
-  GROUP BY namespace, pod_name, container_name, kubernetes_pod_container.tag_id
-) AS container
-GROUP BY namespace, container_name
-"""
-
-
-query_cpu_usage_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  container_name,
-  round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  max(cpu_usage_nanocores),
-  round(avg(cpu_usage_nanocores))
-FROM kubernetes_pod_container
-LEFT OUTER JOIN kubernetes_pod_container_tag
-ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id
-WHERE
-  namespace='{namespace}'
-  AND container_name='{container_name}'
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP BY namespace, pod_name, container_name, timestamp
-"""
-
-
-query_memory_usage_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  container_name,
-  round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  max(memory_usage_bytes) as max,
-  round(avg(memory_usage_bytes)) as avg
-FROM kubernetes_pod_container
-LEFT OUTER JOIN kubernetes_pod_container_tag
-ON kubernetes_pod_container.tag_id = kubernetes_pod_container_tag.tag_id
-WHERE
-  namespace='{namespace}'
-  AND container_name='{container_name}'
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP BY namespace, pod_name, container_name, timestamp
-"""
-
-
-query_network_usage_sql_tpl = """
-SELECT
-  namespace,
-  pod_name,
-  round(EXTRACT(EPOCH FROM time_bucket('{every}', time))) as timestamp,
-  last(rx_bytes, time) - first(rx_bytes, time) as rx_bytes,
-  last(tx_bytes, time) - first(tx_bytes, time) as tx_bytes
-FROM kubernetes_pod_network
-LEFT OUTER JOIN kubernetes_pod_network_tag
-ON kubernetes_pod_network.tag_id = kubernetes_pod_network_tag.tag_id
-WHERE
-  namespace='{namespace}'
-  AND pod_name like '{pod_name_prefix}%'
-  AND time > to_timestamp({start})
-  AND time < to_timestamp({stop})
-GROUP by namespace, pod_name, timestamp
-"""
-
-
-query_loadbalancer_promql_tpl = """
-kube_service_status_load_balancer_ingress{namespace=~"%s"}
-"""
-
-
-def query_loadbalancer(namespaces: Iterator[str],
-                       start: int, stop: int) -> Iterator[Dict[str, str]]:
-    promql = query_loadbalancer_promql_tpl % "|".join(namespaces)
-    params = {"query": promql, "start": start, "end": stop}
-    response = requests.get(
-        urljoin(settings.DRYCC_PROMETHEUS_URL, "/api/v1/query"), params=params)
-    if response.status_code != 200:
-        return StopIteration
-    yield from (metric["metric"] for metric in response.json()["data"]["result"])
-
-
-def query_network_flow(namespaces: Iterator[str],
-                       start: int, stop: int) -> Iterator[tuple[str, str, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        namespace_range = ', '.join([f"'{namespace}'" for namespace in namespaces])
-        sql = query_network_flow_sql_tpl.format(
-            namespace_range=namespace_range, start=start, stop=stop)
-        cursor.execute(sql)
-        yield from cursor
-
-
-def query_container_count(namespace: str, container_type: str, start: int, stop: int) -> int:
-    with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, container_type)
-        sql = query_container_count_sql_tpl.format(
-            namespace=namespace, container_name=container_name, start=start, stop=stop)
-        cursor.execute(sql)
-        row = cursor.fetchone()
-        return row[0] if row else 0
-
-
-def query_cpu_usage(namespace: str, container_type: str,
-                    start: int, stop: int, every: str
-                    ) -> Iterator[tuple[str, str, str, int, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, container_type)
-        sql = query_cpu_usage_sql_tpl.format(
-            namespace=namespace, container_name=container_name,
-            start=start, stop=stop, every=every)
-        cursor.execute(sql)
-        yield from cursor
-
-
-def query_memory_usage(namespace: str, container_type: str,
-                       start: int, stop: int, every: str
-                       ) -> Iterator[tuple[str, str, str, int, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        container_name = "%s-%s" % (namespace, container_type)
-        sql = query_memory_usage_sql_tpl.format(
-            namespace=namespace, container_name=container_name,
-            start=start, stop=stop, every=every)
-        cursor.execute(sql)
-        yield from cursor
-
-
-def query_network_usage(namespace: str, container_type: str,
-                        start: int, stop: int, every: str
-                        ) -> Iterator[tuple[str, str, int, int, int]]:
-    with closing(connections['monitor'].cursor()) as cursor:
-        pod_name_prefix = "%s-%s" % (namespace, container_type)
-        sql = query_network_usage_sql_tpl.format(
-            namespace=namespace, pod_name_prefix=pod_name_prefix,
-            start=start, stop=stop, every=every)
-        cursor.execute(sql)
-        yield from cursor
+from django.template import Template, Context
+
+
+query_last_metrics_promql_tpl = Template("""
+last_over_time({__name__=~"{{metrics}}",namespace="{{namespace}}"}[{{duration}}])
+""")
+
+
+async def query_prom(url, params) -> list[tuple[dict[str, str], int]]:
+    async with aiohttp.ClientSession() as session:
+        async with session.get(url, params=params) as response:
+            if response.status != 200:
+                return []
+            response_json = await response.json()
+            if response_json['status'] != 'success':
+                return []
+            return response_json['data']['result']
+
+
+async def last_metrics(namespace) -> AsyncGenerator[Iterator, str]:
+    if not settings.DRYCC_METRICS_CONFIG:
+        return
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
+    promql = query_last_metrics_promql_tpl.render(Context({
+        "metrics": '|'.join(settings.DRYCC_METRICS_CONFIG.keys()),
+        "namespace": namespace,
+        "duration": settings.DRYCC_METRICS_INTERVAL
+    }))
+    for item in await query_prom(url, {"query": promql, "start": int(time.time() - 60)}):
+        yield '%s{%s} %s\n' % (
+            item['metric']['__name__'],
+            ','.join([
+                f'{key}="{value}"' for key, value in item['metric'].items()
+                if key in settings.DRYCC_METRICS_CONFIG[item['metric']['__name__']]
+            ]),
+            item['value'][1]
+        )
+
+
+async def query_volume_usage(namespaces: Iterator[str], start: int, stop: int
+                             ) -> list[tuple[dict[str, str], int]]:
+    if not settings.DRYCC_VICTORIAMETRICS_URL:
+        return []
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
+    promql = Template(settings.DRYCC_VOLUME_USAGE_TEMPLATE).render(Context({
+        "namespaces": "|".join(namespaces)
+    }))
+    return await query_prom(url, {"query": promql, "start": start, "end": stop})
+
+
+async def query_network_usage(namespaces: Iterator[str], start: int, stop: int
+                              ) -> list[tuple[dict[str, str], int]]:
+    url = f"{settings.DRYCC_VICTORIAMETRICS_URL}/api/v1/query"
+    promql = Template(settings.DRYCC_NETWORK_USAGE_TEMPLATE).render(Context({
+        "namespaces": "|".join(namespaces),
+        "duration": f"{stop-start}s"
+    }))
+    return await query_prom(url, {"query": promql, "start": start, "end": stop})
diff --git a/rootfs/api/oauth.py b/rootfs/api/oauth.py
deleted file mode 100644
index 90d0fb9af..000000000
--- a/rootfs/api/oauth.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from typing import Dict
-
-import requests
-from django.conf import settings
-from authlib.integrations.requests_client import OAuth2Session
-
-
-class OAuthManager(object):
-
-    def __init__(self):
-        self.client_id = settings.SOCIAL_AUTH_DRYCC_KEY
-        self.client_secret = settings.SOCIAL_AUTH_DRYCC_SECRET
-        self.token_url = settings.SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL
-        self.api_url = settings.SOCIAL_AUTH_DRYCC_ACCESS_API_URL
-        self.client = OAuth2Session(self.client_id, self.client_secret)
-
-    def get_user_by_token(self, token: str) -> Dict:
-        response = requests.get(f'{self.api_url}/user/info/', headers={
-            'Authorization': f'Bearer {token}'
-        })
-        result = response.json()
-        return result
-
-    def get_email_by_token(self, token: str) -> Dict:
-        response = requests.get(f'{self.api_url}/user/email/', headers={
-            'Authorization': f'Bearer {token}'
-        })
-        result = response.json()
-        return result
diff --git a/rootfs/api/permissions.py b/rootfs/api/permissions.py
index 4d652324d..4bd7f90d1 100644
--- a/rootfs/api/permissions.py
+++ b/rootfs/api/permissions.py
@@ -1,53 +1,22 @@
 import base64
 from rest_framework import permissions
 from django.conf import settings
-from django.contrib.auth.models import AnonymousUser
 from api import manager
-from api.models.blocklist import Blocklist, App
+from api.models import blocklist
+from api.models.workspace import Workspace, WorkspaceMember
 
 
 def get_app_status(app):
-    blocklist = Blocklist.get_blocklist(app)
-    if blocklist:
-        return False, blocklist.remark
+    block = blocklist.Blocklist.get_blocklist(app)
+    if block:
+        return False, block.remark
     if settings.WORKFLOW_MANAGER_URL:
-        status = manager.User().get_status(app.owner.pk)
+        status = manager.WorkspaceAPI().get_status(app.workspace_id)
         if not status["is_active"]:
             return False, status["message"]
     return True, None
 
 
-def has_app_permission(request, obj):
-    if isinstance(obj, App) or hasattr(obj, 'app'):
-        app = obj if isinstance(obj, App) else obj.app
-        is_ok, message = get_app_status(app)
-        if is_ok:
-            if request.user.is_superuser:
-                return True, None
-            elif app.owner == request.user:
-                return True, None
-            elif request.user.is_staff or request.user.has_perm('use_app', app):
-                if request.method != 'DELETE':
-                    return True, None
-                else:
-                    return False, "User does not have permission to delete"
-        else:
-            return is_ok, message
-    return False, "App object does not exist or does not have permission."
-
-
-class IsAnonymous(permissions.BasePermission):
-    """
-    View permission to allow anonymous users.
-    """
-
-    def has_permission(self, request, view):
-        """
-        Return `True` if permission is granted, `False` otherwise.
-        """
-        return type(request.user) is AnonymousUser
-
-
 class IsOwner(permissions.BasePermission):
     """
     Object-level permission to allow only owners of an object to access it.
@@ -61,70 +30,45 @@ def has_object_permission(self, request, view, obj):
             return False
 
 
-class IsOwnerOrAdmin(permissions.BasePermission):
-    """
-    Object-level permission to allow only owners of an object or administrators to access it.
-    Assumes the model instance has an `owner` attribute.
-    """
-    def has_object_permission(self, request, view, obj):
-        if request.user.is_superuser:
-            return True
-        if hasattr(obj, 'owner'):
-            return obj.owner == request.user
-        else:
-            return False
-
-
 class IsAppUser(permissions.BasePermission):
     """
-    Object-level permission to allow owners or collaborators to access
-    an app-related model.
+    Object-level permission to allow only users who are owners
+    or collaborators of an app to access it.
     """
-    def has_object_permission(self, request, view, obj):
-        return has_app_permission(request, obj)[0]
 
-
-class IsAdmin(permissions.BasePermission):
-    """
-    View permission to allow only admins.
-    """
-
-    def has_permission(self, request, view):
-        """
-        Return `True` if permission is granted, `False` otherwise.
-        """
-        return request.user.is_superuser
-
-
-class IsAdminOrSafeMethod(permissions.BasePermission):
-    """
-    View permission to allow only admins to use unsafe methods
-    including POST, PUT, DELETE.
-
-    This allows
-    """
-
-    def has_permission(self, request, view):
-        """
-        Return `True` if permission is granted, `False` otherwise.
-        """
-        return request.method in permissions.SAFE_METHODS or request.user.is_superuser
+    def has_object_permission(self, request, view, obj):
+        if request.user.is_superuser or request.user.is_staff:
+            return True
+        elif getattr(obj, "user", None) == request.user:
+            return True
+        elif isinstance(obj, Workspace) or hasattr(obj, 'workspace'):
+            workspace = obj if isinstance(obj, Workspace) else obj.workspace
+            if request.method in ["GET", "HEAD", "OPTIONS"]:
+                allowed_roles = ["viewer", "member", "admin"]
+            elif request.method in ["POST", "PUT", "PATCH"]:
+                allowed_roles = ["member", "admin"]
+            else:
+                allowed_roles = ["admin"]
+            return WorkspaceMember.objects.filter(
+                workspace=workspace, user=request.user, role__in=allowed_roles,
+            ).exists()
+        return False
 
 
-class HasBuilderAuth(permissions.BasePermission):
+class IsServiceToken(permissions.BasePermission):
     """
-    View permission to allow builder to perform actions
-    with a special HTTP header
+    The service token is used for internal communication between Drycc components,
+    such as the builder and Quickwit.
     """
 
     def has_permission(self, request, view):
         """
         Return `True` if permission is granted, `False` otherwise.
         """
-        auth_header = request.META.get('HTTP_X_DRYCC_BUILDER_AUTH')
+        auth_header = request.META.get('HTTP_X_DRYCC_SERVICE_KEY')
         if not auth_header:
             return False
-        return auth_header == settings.BUILDER_KEY
+        return auth_header == settings.SERVICE_KEY
 
 
 class IsWorkflowManager(permissions.BasePermission):
diff --git a/rootfs/api/pipeline.py b/rootfs/api/pipeline.py
deleted file mode 100644
index 0426ccb71..000000000
--- a/rootfs/api/pipeline.py
+++ /dev/null
@@ -1,22 +0,0 @@
-from api.serializers import UserSerializer
-
-
-def update_or_create(backend, user, response, *args, **kwargs):
-    user, created = UserSerializer.update_or_create(response)
-
-    if not created:
-        return {'is_new': False}
-
-    return {
-        'is_new': True,
-        'user': user
-    }
-
-
-def load_extra_data(backend, details, response, uid, user, *args, **kwargs):
-    social = kwargs.get('social') or \
-             backend.strategy.storage.user.get_social_auth(backend.name, uid)
-    if social:
-        extra_data = backend.extra_data(user, uid, response, details,
-                                        *args, **kwargs)
-        social.set_extra_data(extra_data)
diff --git a/rootfs/api/routers.py b/rootfs/api/routers.py
index 801d6c95c..3e5e98115 100644
--- a/rootfs/api/routers.py
+++ b/rootfs/api/routers.py
@@ -32,4 +32,8 @@ def allow_relation(self, obj1, obj2, **hints):
         return True
 
     def allow_migrate(self, db, app_label, model_name=None, **hints):
+        if 'replica' in settings.DATABASES and 'model' in hints:
+            model = hints['model']
+            tracker_key = ".".join([model.__module__, model.__name__])
+            setattr(self._tracker, tracker_key, 'default')
         return True
diff --git a/rootfs/api/routing.py b/rootfs/api/routing.py
index 9864002a1..6806b2fa0 100644
--- a/rootfs/api/routing.py
+++ b/rootfs/api/routing.py
@@ -3,11 +3,17 @@
 
 from . import consumers
 
+http_urlpatterns = [
+    re_path(
+        r'^v2/apps/(?P<id>([\w-]*))/volumes/(?P<name>([\w-]*))/filer/(?P<path>.*)$',
+        consumers.FilerProxyConsumer.as_asgi()),
+]
+
 websocket_urlpatterns = [
     re_path(
-        r'^v2/apps/(?P<id>.*)/logs/?$',
-        consumers.AppLogsConsumer.as_asgi()),
+        r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<name>([\w-]*))/logs/?$',
+        consumers.AppPodLogsConsumer.as_asgi()),
     re_path(
-        r'^v2/apps/(?P<id>.*)/pods/(?P<pod_id>.*)/exec/?$',
-        consumers.AppPodExecConsumer.as_asgi())
+        r'^v2/apps/(?P<id>([\w-]*))/pods/(?P<name>([\w-]*))/exec/?$',
+        consumers.AppPodExecConsumer.as_asgi()),
 ]
diff --git a/rootfs/api/schemas/rules.py b/rootfs/api/schemas/rules.py
deleted file mode 100644
index 97f9f12e0..000000000
--- a/rootfs/api/schemas/rules.py
+++ /dev/null
@@ -1,211 +0,0 @@
-HTTP_HEADER_FILTER_SCHEMA = {
-    "set": {
-        "type": "array",
-        "items": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string"},
-                "value": {"type": "string"}
-            }
-        }
-    },
-    "add": {
-        "type": "array",
-        "items": {
-            "type": "object",
-            "properties": {
-                "name": {"type": "string"},
-                "value": {"type": "string"}
-            }
-        }
-    },
-    "remove": {"type": "array"}
-}
-
-FILTER_SCHEMA = {
-    # Type identifies the type of filter to apply.
-    # As with other API fields, types are classified into three conformance levels:
-    "type": {
-        "type": "string",
-        "enum": ["ExtensionRef", "RequestHeaderModifier", "RequestMirror", "RequestRedirect", "ResponseHeaderModifier", "URLRewrite"],  # noqa
-    },
-    # ExtensionRef is an optional, implementation-specific extension to the “filter” behavior. # noqa
-    # For example, resource “myroutefilter” in group “networking.example.net”).
-    # ExtensionRef MUST NOT be used for core and extended filters.
-    "extensionRef": {
-        "type": "object",
-        "properties": {
-            "group": {"type": "string"},
-            "kind": {"type": "string"},
-            "name": {"type": "string"}
-        },
-        "required": ["group", "kind", "name"],
-        "additionalProperties": False
-    },
-    # RequestHeaderModifier defines a schema for a filter that modifies request headers.
-    "requestHeaderModifier": {
-        "type": "object",
-        "properties": HTTP_HEADER_FILTER_SCHEMA,
-        "additionalProperties": False
-    },
-    # ResponseHeaderModifier defines a schema for a filter that modifies response headers.
-    "responseHeaderModifier": {
-        "type": "object",
-        "properties": HTTP_HEADER_FILTER_SCHEMA,
-        "additionalProperties": False
-    },
-    # RequestMirror defines a schema for a filter that mirrors requests.
-    # Requests are sent to the specified destination, but responses from that destination are ignored. # noqa
-    "requestMirror": {
-        "type": "object",
-        "properties": {
-            "backendRef": {
-                "properties": {
-                    "group": {"type": "string"},
-                    "kind": {"type": "string"},
-                    "name": {"type": "string"},
-                    "namespace": {"type": "string"},
-                    "port": {"type": "integer"},
-                },
-                "required": ["name"],
-                "additionalProperties": False
-            },
-        },
-        "required": ["backendRef"],
-        "additionalProperties": False
-    },
-    # RequestRedirect defines a schema for a filter that responds to the request with an HTTP redirection. # noqa
-    "requestRedirect": {
-        "type": "object",
-        "properties": {
-            "scheme": {"type": "string"},
-            "hostname": {"type": "string"},
-            "path": {"type": "string"},
-            "port": {"type": "integer"},
-            "statusCode": {"type": "integer"}
-        }
-    },
-    # URLRewrite defines a schema for a filter that modifies a request during forwarding
-    "urlRewrite": {
-        "hostname": {"type": "string"},
-        "path": {"type": "string"},
-    }
-}
-
-RULES_SCHEMA = {
-    "type": "array",
-    "items": {
-        "properties": {
-            # Matches define conditions used for matching the rule against incoming HTTP requests. # noqa
-            # Each match is independent, i.e. this rule will be matched if any one of the matches is satisfied. # noqa
-            # More info: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteMatch # noqa
-            "matches": {
-                "type": "array",
-                "items": {
-                    "properties": {
-                        # Path specifies a HTTP request path matcher.
-                        # If this field is not specified, a default prefix match on the “/” path is provided. # noqa
-                        "path": {
-                            "type": "object",
-                            "properties": {
-                                "type": {
-                                    "type": "string",
-                                    "enum": ["Exact", "PathPrefix", "RegularExpression"],
-                                    "default": "PathPrefix"
-                                },
-                                "value": {"type": "string"}
-                            },
-                            "additionalProperties": False
-                        },
-                        # Headers specifies HTTP request header matchers. Multiple match values are ANDed together, # noqa
-                        # meaning, a request must match all the specified headers to select the route. # noqa
-                        # gateway.networking.k8s.io/v1beta1.HTTPHeaderMatch
-                        # More info: https: // gateway-api.sigs.k8s.io/references/spec /
-                        "headers": {
-                            "type": "array",
-                            "items": {
-                                "properties": {
-                                    "type": {
-                                        "type": "string",
-                                        "enum": ["Exact", "RegularExpression"],
-                                        "default": "Exact"
-                                    },
-                                    "name": {"type": "string"},
-                                    "value": {"type": "string"}
-                                },
-                                "additionalProperties": False
-                            }
-                        },
-                        # QueryParams specifies HTTP query parameter matchers. Multiple match values are ANDed together, # noqa
-                        # meaning, a request must match all the specified query parameters to select the route. # noqa
-                        "queryParams": {
-                            "type": "array",
-                            "items": {
-                                "properties": {
-                                    "type": {
-                                        "type": "string",
-                                        "enum": ["Exact", "RegularExpression"],
-                                        "default": "Exact"
-                                    },
-                                    "name": {"type": "string"},
-                                    "value": {"type": "string"}
-                                },
-                                "additionalProperties": False
-                            }
-                        },
-                        "method": {
-                            "type": "string",
-                            "enum": ["CONNECT", "DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT", "TRACE"],  # noqa
-                        }
-                    },
-                    "additionalProperties": False
-                }
-            },
-            # Filters define the filters that are applied to requests that match this rule.
-            # More info: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPRouteFilter # noqa
-            "filters": {
-                "type": "array",
-                "items": {
-                    "properties": FILTER_SCHEMA,
-                    "additionalProperties": False
-                },
-            },
-            # BackendRefs defines the backend(s) where matching requests should be sent.
-            # More info: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.HTTPBackendRef # noqa
-            "backendRefs": {
-                "type": "array",
-                "items": {
-                    "properties": {
-                        "filters": {
-                            "type": "array",
-                            "items": {
-                                "properties": FILTER_SCHEMA,
-                                "additionalProperties": False
-                            },
-                        },
-                        "group": {"type": "string"},
-                        "kind": {"type": "string"},
-                        "name": {"type": "string"},
-                        "namespace": {"type": "string"},
-                        "port": {"type": "integer"},
-                        "weight": {"type": "integer"}
-                    },
-                    "required": ["name"],
-                    "additionalProperties": False
-                }
-            }
-        },
-        "required": ["backendRefs"],
-        "additionalProperties": False
-    }
-}
-
-SCHEMA = {
-    "$schema": "http://json-schema.org/draft-07/schema#",
-    "properties": {
-        "canary": RULES_SCHEMA,
-        "stable": RULES_SCHEMA,
-    },
-    "required": ["canary", "stable"],
-    "additionalProperties": False
-}
diff --git a/rootfs/api/serializers.py b/rootfs/api/serializers/__init__.py
similarity index 51%
rename from rootfs/api/serializers.py
rename to rootfs/api/serializers/__init__.py
index c604ddc87..146736d93 100644
--- a/rootfs/api/serializers.py
+++ b/rootfs/api/serializers/__init__.py
@@ -4,42 +4,96 @@
 import time
 import json
 import logging
-import jmespath
 import re
-import jsonschema
 import idna
-from urllib.parse import urlparse
 
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.utils import timezone
+from django.utils.translation import gettext_lazy
 from rest_framework import serializers
 
+
 from api import models
+from api.utils import validate_json, validate_reserved_names
 from api.exceptions import DryccException
-from api.schemas.rules import SCHEMA as RULES_SCHEMA
-from api.schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
-from api.schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
+from api.models.volume import Volume
+from api.models.base import PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH
+from scheduler.resources.pod import DEFAULT_CONTAINER_PORT
+from .schemas import rules
+from .schemas.config import SCHEMA as CONFIG_SCHEMA
+from .schemas.volumes import SCHEMA as VOLUMES_SCHEMA
+from .schemas.autoscale import SCHEMA as AUTOSCALE_SCHEMA
+from .schemas.healthcheck import SCHEMA as HEALTHCHECK_SCHEMA
+from .schemas.dryccfile import (SCHEMA as DRYCCFILE_SCHEMA, PROCTYPE_REGEX, CONFIGKEY_REGEX)
 
 
 User = get_user_model()
 logger = logging.getLogger(__name__)
 SERVICE_PROTOCOL_MATCH = re.compile(r'^(TCP|UDP|SCTP)$')
-SERVICE_PROTOCOL_MISMATCH_MSG = "the service protocol only supports TCP, UDP, and SCTP"
+SERVICE_PROTOCOL_MISMATCH_MSG = (
+    "the service protocol only supports: %s" % SERVICE_PROTOCOL_MATCH.pattern)
 GATEWAY_PROTOCOL_MATCH = re.compile(r'^(HTTP|HTTPS|TCP|TLS|UDP)$')
-GATEWAY_PROTOCOL_MISMATCH_MSG = "the gateway protocol only supports HTTP, HTTPS, TCP, TLS and UDP"
+GATEWAY_PROTOCOL_MISMATCH_MSG = (
+    "the gateway protocol only supports: %s" % GATEWAY_PROTOCOL_MATCH.pattern)
 ROUTE_PROTOCOL_MATCH = re.compile(r'^(HTTPRoute|TCPRoute|UDPRoute|TLSRoute)$')
-ROUTE_PROTOCOL_MISMATCH_MSG = "the route kind only supports HTTPRoute, TCPRoute, UDPRoute, and TLSRoute"  # noqa
-PROCTYPE_MATCH = re.compile(r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)(?<!-canary)$')
-PROCTYPE_MISMATCH_MSG = "Process types can only contain lowercase alphanumeric characters"
-MEMLIMIT_MATCH = re.compile(r'^(?P<mem>([1-9][0-9]*[mgMG]))$', re.IGNORECASE)
-CPUSHARE_MATCH = re.compile(r'^(?P<cpu>([-+]?[1-9][0-9]*[m]?))$')
+ROUTE_PROTOCOL_MISMATCH_MSG = (
+    "the route kind only supports: %s" % ROUTE_PROTOCOL_MATCH.pattern)
+PROCTYPE_MATCH = re.compile(PROCTYPE_REGEX)
+PROCTYPE_MISMATCH_MSG = "Process types can only supports: %s" % PROCTYPE_MATCH.pattern
+
 TAGVAL_MATCH = re.compile(r'^(?:[a-zA-Z\d][-\.\w]{0,61})?[a-zA-Z\d]$')
-CONFIGKEY_MATCH = re.compile(r'^[a-z_]+[a-z0-9_]*$', re.IGNORECASE)
+CONFIGKEY_MATCH = re.compile(CONFIGKEY_REGEX, re.IGNORECASE)
+CONFIGKEY_MISMATCH_MSG = ("Config key can only supports: %s" % CONFIGKEY_MATCH.pattern)
+CONFIG_LIMITS_MISMATCH_MSG = "The limit plan {} does not exist"
+
 TERMINATION_GRACE_PERIOD_MATCH = re.compile(r'^[0-9]*$')
+TERMINATION_GRACE_PERIOD_MISMATCH_MSG = (
+    "Termination Grace Period format: %s" % TERMINATION_GRACE_PERIOD_MATCH.pattern)
+VOLUME_TYPE_MATCH = re.compile(r'^(%s)$' % "|".join([v[0] for v in Volume.TYPE_CHOICES]))
+VOLUME_TYPE_MISMATCH_MSG = "Volume type pattern: %s" % VOLUME_TYPE_MATCH.pattern
 VOLUME_SIZE_MATCH = re.compile(r'^(?P<volume>([1-9][0-9]*[gG]))$', re.IGNORECASE)
-VOLUME_PATH = re.compile(r'^\/(\w+\/?)+$', re.IGNORECASE)
-METRIC_EVERY = re.compile(r'^[1-9][0-9]*m$')
+VOLUME_SIZE_MISMATCH_MSG = (
+    "Volume size limit format: <number><unit> or <number><unit>/<number><unit>, "
+    "where unit = G, range: %sG~%sG"
+) % (settings.KUBERNETES_LIMITS_MAX_VOLUME, settings.KUBERNETES_LIMITS_MIN_VOLUME)
+VOLUME_PATH_MATCH = re.compile(r'^\/([\w]+[\w-]*\/?)+$', re.IGNORECASE)
+METRIC_EVERY_MATCH = re.compile(r'^[1-9][0-9]*m$')
+HEALTHCHECK_MATCH = re.compile(r'^(livenessProbe|readinessProbe|startupProbe)$')
+HEALTHCHECK_MISMATCH_MSG = "Healthcheck pattern: %s" % HEALTHCHECK_MATCH.pattern
+
+
+def validate_port(value):
+    if not str(value).isnumeric():
+        raise serializers.ValidationError('port can only be a numeric value')
+    elif int(value) not in range(1, 65536):
+        raise serializers.ValidationError('port needs to be between 1 and 65535')
+    return value
+
+
+def validate_ptype(value):
+    if not re.match(PROCTYPE_MATCH, value):
+        raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+    if len(value) < PTYPE_MIN_LENGTH or len(value) > PTYPE_MAX_LENGTH:
+        raise serializers.ValidationError(
+            "The length of ptype must be between {} and {}".format(
+                PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH))
+    validate_reserved_names(value)
+    return value
+
+
+def validate_name(value):
+    NAME_REGEX = r'^(?P<name>{})$'.format(settings.NAME_REGEX)
+    NAME_MATCH = re.compile(NAME_REGEX)
+    NAME_MISMATCH_MSG = "name can only supports: %s" % NAME_MATCH.pattern
+    if not re.match(NAME_REGEX, value):
+        raise serializers.ValidationError(NAME_MISMATCH_MSG)
+    if len(value) < PTYPE_MIN_LENGTH or len(value) > PTYPE_MAX_LENGTH:
+        raise serializers.ValidationError(
+            "The length of name must be between {} and {}".format(
+                PTYPE_MIN_LENGTH, PTYPE_MAX_LENGTH))
+    validate_reserved_names(value)
+    return value
 
 
 class JSONFieldSerializer(serializers.JSONField):
@@ -57,26 +111,46 @@ def to_internal_value(self, data):
 
     def to_representation(self, obj):
         """Serialize the field's JSON data, for read operations."""
-        for k, v in obj.items():
-            if v is None:  # NoneType is used to unset a value
-                continue
-
-            try:
-                if self.convert_to_str:
-                    obj[k] = str(v)
-            except ValueError:
-                obj[k] = v
-                # Do nothing, the validator will catch this later
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                if v is None:  # NoneType is used to unset a value
+                    continue
 
+                try:
+                    if isinstance(v, (dict, list)):
+                        self.to_representation(v)
+                    elif self.convert_to_str:
+                        obj[k] = str(v)
+                except ValueError:
+                    obj[k] = v
+                    # Do nothing, the validator will catch this later
+        elif isinstance(obj, list):
+            for i in range(len(obj)):
+                obj[i] = self.to_representation(obj[i])
         return obj
 
 
+class AuthSerializer(serializers.Serializer):
+    username = serializers.CharField(
+        label=gettext_lazy("Username"),
+        required=False,
+        write_only=True
+    )
+    password = serializers.CharField(
+        label=gettext_lazy("Password"),
+        style={'input_type': 'password'},
+        required=False,
+        trim_whitespace=False,
+        write_only=True,
+    )
+
+
 class UserSerializer(serializers.ModelSerializer):
     class Meta:
         model = User
-        fields = ['email', 'username', 'password', 'first_name', 'last_name', 'is_superuser',
-                  'is_staff', 'groups', 'user_permissions', 'last_login', 'date_joined',
-                  'is_active']
+        fields = ['id', 'email', 'username', 'password', 'first_name', 'last_name',
+                  'is_superuser', 'is_staff', 'groups', 'user_permissions', 'last_login',
+                  'date_joined', 'is_active']
         read_only_fields = ['id', 'is_superuser', 'is_staff', 'groups',
                             'user_permissions', 'last_login', 'date_joined', 'is_active']
         extra_kwargs = {'password': {'write_only': True}}
@@ -104,6 +178,17 @@ def update_or_create(data):
         return user, created
 
 
+class TokenSerializer(serializers.ModelSerializer):
+    """Serialize a :class:`~api.models.base.Token` model."""
+
+    owner = serializers.ReadOnlyField(source='owner.username')
+
+    class Meta:
+        model = models.base.Token
+        fields = ['uuid', 'owner', 'alias', 'fuzzy_key', 'created', 'updated']
+        read_only_fields = fields
+
+
 class AdminUserSerializer(serializers.ModelSerializer):
     """Serialize admin status for a User model."""
 
@@ -116,27 +201,29 @@ class Meta:
 class AppSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.app.App` model."""
 
-    owner = serializers.ReadOnlyField(source='owner.username')
+    workspace = serializers.SlugRelatedField(
+        slug_field='name',
+        queryset=models.workspace.Workspace.objects.all(),
+    )
     structure = serializers.JSONField(required=False)
-    procfile_structure = serializers.JSONField(required=False)
 
     class Meta:
         """Metadata options for a :class:`AppSerializer`."""
         model = models.app.App
-        fields = ['uuid', 'id', 'owner', 'structure', 'procfile_structure', 'created', 'updated']
+        fields = ['uuid', 'id', 'workspace', 'structure', 'created', 'updated']
 
 
 class BuildSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.build.Build` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     procfile = serializers.JSONField(required=False)
+    dryccfile = serializers.JSONField(required=False)
 
     class Meta:
         """Metadata options for a :class:`BuildSerializer`."""
         model = models.build.Build
-        fields = ['owner', 'app', 'image', 'stack', 'sha', 'procfile',
+        fields = ['app', 'image', 'stack', 'sha', 'procfile', 'dryccfile',
                   'dockerfile', 'created', 'updated', 'uuid']
 
     @staticmethod
@@ -144,23 +231,57 @@ def validate_procfile(data):
         for key, value in data.items():
             if value is None or value == "":
                 raise serializers.ValidationError("Command can't be empty for process type")
+            validate_ptype(key)
+        return data
 
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
+    @staticmethod
+    def validate_dryccfile(data):
+        ptypes = set()
+        if data:
+            validate_json(data, DRYCCFILE_SCHEMA, serializers.ValidationError)
+            for pipeline in data['pipeline'].values():
+                if pipeline['ptype'] in ptypes:
+                    raise serializers.ValidationError("Pipeline's ptype must be unique")
+                else:
+                    validate_ptype(pipeline['ptype'])
+                    ptypes.add(pipeline['ptype'])
         return data
 
 
+class LimitSpecSerializer(serializers.ModelSerializer):
+    id = serializers.CharField(required=True)
+    cpu = serializers.JSONField(required=True)
+    memory = serializers.JSONField(required=True)
+    features = serializers.JSONField(required=True)
+    disabled = serializers.BooleanField(required=True)
+
+    class Meta:
+        """Metadata options for a :class:`LimitSpecSerializer`."""
+        model = models.limit.LimitSpec
+        fields = ['id', 'cpu', 'memory', 'features', 'disabled']
+
+
+class LimitPlanSerializer(serializers.ModelSerializer):
+    id = serializers.CharField(required=True)
+    spec = LimitSpecSerializer(required=True)
+    cpu = serializers.IntegerField(required=True)
+    memory = serializers.IntegerField(required=True)
+    features = serializers.JSONField(required=True)
+    disabled = serializers.BooleanField(required=True)
+
+    class Meta:
+        """Metadata options for a :class:`LimitPlanSerializer`."""
+        model = models.limit.LimitSpec
+        fields = ['id', 'spec', 'cpu', 'memory', 'features', 'disabled']
+
+
 class ConfigSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.config.Config` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     values = JSONFieldSerializer(required=False, binary=True)
-    memory = JSONFieldSerializer(required=False, binary=True)
-    lifecycle_post_start = JSONFieldSerializer(required=False, binary=True)
-    lifecycle_pre_stop = JSONFieldSerializer(required=False, binary=True)
-    cpu = JSONFieldSerializer(required=False, binary=True)
+    limits = JSONFieldSerializer(required=False, binary=True)
+    lifecycle = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
     tags = JSONFieldSerializer(required=False, binary=True)
     registry = JSONFieldSerializer(required=False, binary=True)
     healthcheck = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
@@ -174,105 +295,45 @@ class Meta:
 
     @staticmethod
     def validate_values(data):
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
+        validate_json(data, CONFIG_SCHEMA, serializers.ValidationError)
+        for item in data:
+            if (
+                not re.match(CONFIGKEY_MATCH, item['name']) or
+                (item.get('group') and not re.match(CONFIGKEY_MATCH, item['group']))
+            ):
+                raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
+            if item['value'] is None:  # use NoneType to unset an item
                 continue
-
-            if not re.match(CONFIGKEY_MATCH, key):
-                raise serializers.ValidationError(
-                    "Config keys must start with a letter or underscore and "
-                    "only contain [A-z0-9_]")
-
             # Validate PORT
-            if key == 'PORT':
-                if not str(value).isnumeric():
+            if item['name'] == 'PORT':
+                if not str(item['value']).isnumeric():
                     raise serializers.ValidationError('PORT can only be a numeric value')
-                elif int(value) not in range(1, 65536):
+                elif int(item['value']) not in range(1, 65536):
                     # check if hte port is between 1 and 65535. One extra added for range()
                     # http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_serviceport
                     raise serializers.ValidationError('PORT needs to be between 1 and 65535')
-
-            # Validate HEALTHCHECK_*
-            if key == 'HEALTHCHECK_URL':
-                # Only Path information is supported, not query / anchor or anything else
-                # Path is the only thing Kubernetes supports right now
-                # See https://github.com/drycc/controller/issues/774
-                uri = urlparse(value)
-
-                if not uri.path:
-                    raise serializers.ValidationError(
-                        '{} is missing a URI path (such as /healthz). '
-                        'Without it no health check can be done'.format(key)
-                    )
-
-                # Disallow everything but path
-                # https://docs.python.org/3/library/urllib.parse.html
-                if uri.query or uri.fragment or uri.netloc:
-                    raise serializers.ValidationError(
-                        '{} can only be a URI path (such as /healthz) that does not contain '
-                        'other things such as query params'.format(key)
-                    )
-            elif key.startswith('HEALTHCHECK_') and not str(value).isnumeric():
-                # all other healthchecks are integers
-                raise serializers.ValidationError('{} can only be a numeric value'.format(key))
-
         return data
 
     @staticmethod
-    def validate_memory(data):
-        max_memory = settings.KUBERNETES_LIMITS_MAX_MEMORY
-        # The minimum limit memory is equal to the memory allocated by default
-        min_memory = settings.KUBERNETES_LIMITS_MIN_MEMORY
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
-                continue
-
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-            if not re.match(MEMLIMIT_MATCH, str(value)):
-                raise serializers.ValidationError(
-                    "Memory limit format: <number><unit>, "
-                    "where unit = M or G")
-            range_error = "Memory setting is not in allowed range: %sM~%sM" % (
-                min_memory, max_memory)
-            memory_size = int(value[:-1]) * 1024 if value.endswith("G") else int(value[:-1])
-            if memory_size < min_memory or memory_size > max_memory:
-                raise serializers.ValidationError(range_error)
-        return data
-
-    @staticmethod
-    def validate_cpu(data):
-        max_cpu = settings.KUBERNETES_LIMITS_MAX_CPU
-        # The minimum CPU limit is equal to the CPU allocated by default
-        min_cpu = settings.KUBERNETES_LIMITS_MIN_CPU
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
-                continue
-
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-            shares = re.match(CPUSHARE_MATCH, str(value))
-            if not shares:
-                raise serializers.ValidationError(
-                    "CPU limit format: <value>, where value must be a numeric")
-            range_error = "CPU setting is not in allowed range: %sm~%sm" % (
-                min_cpu, max_cpu)
-            cpu_size = int(value) * 1000 if value.isdigit() else int(value[:-1])
-            if cpu_size < min_cpu or cpu_size > max_cpu:
-                raise serializers.ValidationError(range_error)
+    def validate_limits(data):
+        req_plan_ids = []
+        for ptype, plan_id in data.items():
+            validate_ptype(ptype)
+            if plan_id is not None:
+                req_plan_ids.append(plan_id)
+        plan_ids = [plan.id for plan in models.limit.LimitPlan.objects.filter(
+            disabled=False, id__in=req_plan_ids)]
+        for req_plan_id in req_plan_ids:
+            if req_plan_id not in plan_ids:
+                raise serializers.ValidationError(CONFIG_LIMITS_MISMATCH_MSG.format(req_plan_id))
         return data
 
     @staticmethod
     def validate_termination_grace_period(data):
-        for key, value in data.items():
+        for ptype, value in data.items():
+            validate_ptype(ptype)
             if value is None:  # use NoneType to unset an item
                 continue
-
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
             timeout = re.match(TERMINATION_GRACE_PERIOD_MATCH, str(value))
             if not timeout:
                 raise serializers.ValidationError(
@@ -282,78 +343,76 @@ def validate_termination_grace_period(data):
 
     @staticmethod
     def validate_tags(data):
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
-                continue
-
-            # split key into a prefix and name
-            if '/' in key:
-                prefix, name = key.split('/')
-            else:
-                prefix, name = None, key
+        for ptype, values in data.items():
+            validate_ptype(ptype)
+            for key, value in values.items():
+                if value is None:  # use NoneType to unset an item
+                    continue
 
-            # validate optional prefix
-            if prefix:
-                if len(prefix) > 253:
-                    raise serializers.ValidationError(
-                        "Tag key prefixes must 253 characters or less.")
+                # split key into a prefix and name
+                if '/' in key:
+                    prefix, name = key.split('/')
+                else:
+                    prefix, name = None, key
 
-                for part in prefix.split('/'):
-                    if not re.match(TAGVAL_MATCH, part):
+                # validate optional prefix
+                if prefix:
+                    if len(prefix) > 253:
                         raise serializers.ValidationError(
-                            "Tag key prefixes must be DNS subdomains.")
+                            "Tag key prefixes must 253 characters or less.")
 
-            # validate required name
-            if not re.match(TAGVAL_MATCH, name):
-                raise serializers.ValidationError(
-                    "Tag keys must be alphanumeric or \"-_.\", and 1-63 characters.")
+                    for part in prefix.split('/'):
+                        if not re.match(TAGVAL_MATCH, part):
+                            raise serializers.ValidationError(
+                                "Tag key prefixes must be DNS subdomains.")
 
-            # validate value if it isn't empty
-            if value and not re.match(TAGVAL_MATCH, str(value)):
-                raise serializers.ValidationError(
-                    "Tag values must be alphanumeric or \"-_.\", and 1-63 characters.")
+                # validate required name
+                if not re.match(TAGVAL_MATCH, name):
+                    raise serializers.ValidationError(
+                        "Tag keys must be alphanumeric or \"-_.\", and 1-63 characters.")
+
+                # validate value if it isn't empty
+                if value and not re.match(TAGVAL_MATCH, str(value)):
+                    raise serializers.ValidationError(
+                        "Tag values must be alphanumeric or \"-_.\", and 1-63 characters.")
+        return data
 
+    @staticmethod
+    def validate_values_refs(data):
+        for ptype, values in data.items():
+            validate_ptype(ptype)
+            for value in values:
+                if value is None:
+                    continue
+                if not re.match(CONFIGKEY_MATCH, value):
+                    raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
         return data
 
     @staticmethod
     def validate_registry(data):
-        for key, value in data.items():
-            if value is None:  # use NoneType to unset an item
+        for ptype, values in data.items():
+            validate_ptype(ptype)
+            if not values:
                 continue
-
-            if not re.match(CONFIGKEY_MATCH, key):
-                raise serializers.ValidationError(
-                    "Config keys must start with a letter or underscore and "
-                    "only contain [A-z0-9_]")
-
+            for key, value in values.items():
+                if value is None:
+                    continue
+                if not re.match(CONFIGKEY_MATCH, key):
+                    raise serializers.ValidationError(CONFIGKEY_MISMATCH_MSG)
         return data
 
     @staticmethod
     def validate_healthcheck(data):
-        for procType, healthcheck in data.items():
+        for ptype, healthcheck in data.items():
+            validate_ptype(ptype)
             if healthcheck is None:
                 continue
             for key, value in healthcheck.items():
                 if value is None:
                     continue
-                if key not in ['livenessProbe', 'readinessProbe']:
-                    raise serializers.ValidationError(
-                        "Healthcheck keys must be either livenessProbe or readinessProbe")
-                try:
-                    jsonschema.validate(value, HEALTHCHECK_SCHEMA)
-                except jsonschema.ValidationError as e:
-                    raise serializers.ValidationError(
-                        "could not validate {}: {}".format(value, e.message))
-
-            # http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_probe
-            # liveness only supports successThreshold=1, no other value
-            # This is not in the schema since readiness supports other values
-            threshold = jmespath.search('livenessProbe.successThreshold', healthcheck)
-            if threshold is not None and threshold != 1:
-                raise serializers.ValidationError(
-                    'livenessProbe successThreshold can only be 1'
-                )
-
+                if not re.match(HEALTHCHECK_MATCH, key):
+                    raise serializers.ValidationError(HEALTHCHECK_MISMATCH_MSG)
+                validate_json(value, HEALTHCHECK_SCHEMA, serializers.ValidationError)
         return data
 
 
@@ -361,7 +420,6 @@ class ReleaseSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.release.Release` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
 
     class Meta:
         """Metadata options for a :class:`ReleaseSerializer`."""
@@ -384,12 +442,11 @@ class DomainSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.domain.Domain` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
 
     class Meta:
         """Metadata options for a :class:`DomainSerializer`."""
         model = models.domain.Domain
-        fields = ['owner', 'created', 'updated', 'app', 'domain']
+        fields = ['created', 'updated', 'app', 'domain', 'ptype']
         read_only_fields = ['uuid']
 
     @staticmethod
@@ -439,34 +496,29 @@ def ToACE(x): return idna.alabel(x).decode("utf-8", "strict")
 
         return aceValue
 
+    validate_ptype = staticmethod(validate_ptype)
+
 
 class ServiceSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.service.Service` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
-    port = serializers.IntegerField(default=5000)
-    protocol = serializers.CharField(default="TCP")
-    target_port = serializers.IntegerField(default=5000)
-    procfile_type = serializers.CharField(required=True)
+    port = serializers.IntegerField(required=True)
+    protocol = serializers.CharField(required=True)
+    target_port = serializers.IntegerField(default=DEFAULT_CONTAINER_PORT)
+    ptype = serializers.CharField(required=True)
 
     class Meta:
         """Metadata options for a :class:`ServiceSerializer`."""
         model = models.service.Service
-        fields = ['owner', 'created', 'updated', 'app', 'procfile_type']
+        fields = ['created', 'updated', 'app', 'ptype']
         read_only_fields = ['uuid']
 
-    @staticmethod
-    def validate_port(value):
-        if not str(value).isnumeric():
-            raise serializers.ValidationError('port can only be a numeric value')
-        elif int(value) not in range(1, 65536):
-            raise serializers.ValidationError('port needs to be between 1 and 65535')
-        return value
-
     @staticmethod
     def validate_protocol(value):
-        if not re.match(SERVICE_PROTOCOL_MATCH, value):
+        if value is None or value == "":
+            value = "TCP"
+        elif not re.match(SERVICE_PROTOCOL_MATCH, value):
             raise serializers.ValidationError(SERVICE_PROTOCOL_MISMATCH_MSG)
         return value
 
@@ -474,18 +526,14 @@ def validate_protocol(value):
     def validate_target_port(cls, value):
         return cls.validate_port(value)
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-        return value
+    validate_port = staticmethod(validate_port)
+    validate_ptype = staticmethod(validate_ptype)
 
 
 class CertificateSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.certificate.Certificate` model."""
 
-    owner = serializers.ReadOnlyField(source='owner.username')
+    app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
     domains = serializers.ReadOnlyField()
     san = serializers.ListField(
         child=serializers.CharField(allow_blank=True, allow_null=True, required=False),
@@ -509,6 +557,29 @@ class PodSerializer(serializers.BaseSerializer):
     type = serializers.CharField()
     release = serializers.CharField(required=False)
     started = serializers.DateTimeField(required=False)
+    ready = serializers.CharField()
+    restarts = serializers.IntegerField()
+
+    def to_representation(self, obj):
+        return obj
+
+
+class EventSerializer(serializers.BaseSerializer):
+    reason = serializers.CharField()
+    message = serializers.CharField()
+    created = serializers.DateTimeField(required=False)
+
+    def to_representation(self, obj):
+        return obj
+
+
+class PtypeSerializer(serializers.BaseSerializer):
+    name = serializers.CharField(required=False)
+    release = serializers.CharField()
+    ready = serializers.CharField()
+    up_to_date = serializers.CharField()
+    available_replicas = serializers.CharField(required=False)
+    started = serializers.DateTimeField(required=False)
 
     def to_representation(self, obj):
         return obj
@@ -518,8 +589,6 @@ class AppSettingsSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.appsettings.AppSettings` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
-    canaries = serializers.JSONField(required=False)
     autoscale = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
     label = JSONFieldSerializer(convert_to_str=False, required=False, binary=True)
 
@@ -530,15 +599,11 @@ class Meta:
 
     @staticmethod
     def validate_autoscale(data):
-        for _, autoscale in data.items():
+        for ptype, autoscale in data.items():
+            validate_ptype(ptype)
             if autoscale is None:
                 continue
-            try:
-                jsonschema.validate(autoscale, AUTOSCALE_SCHEMA)
-            except jsonschema.ValidationError as e:
-                raise serializers.ValidationError(
-                    "could not validate {}: {}".format(autoscale, e.message)
-                )
+            validate_json(autoscale, AUTOSCALE_SCHEMA, serializers.ValidationError)
         return data
 
 
@@ -546,7 +611,7 @@ class TLSSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.tls.TLS` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
+    events = serializers.ReadOnlyField()
 
     class Meta:
         """Metadata options for a :class:`AppTLSSerializer`."""
@@ -558,44 +623,40 @@ class VolumeSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.volume.Volume` model."""
 
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
-    name = serializers.CharField()
-    size = serializers.CharField()
     path = JSONFieldSerializer(required=False, binary=True)
+    parameters = serializers.JSONField(required=False)
 
     class Meta:
         """Metadata options for a :class:`AppVolumeSerializer`."""
         model = models.volume.Volume
         fields = '__all__'
 
-    @staticmethod
-    def validate_size(data):
+    def validate_size(self, data):
+        # check size format
         if not re.match(VOLUME_SIZE_MATCH, data):
-            raise serializers.ValidationError(
-                "Volume size limit format: <number><unit> or <number><unit>/<number><unit>, "
-                "where unit = G")
+            raise serializers.ValidationError(VOLUME_SIZE_MISMATCH_MSG)
+        # check volume size
+        volume_size = int(data[:-1])
         max_volume = settings.KUBERNETES_LIMITS_MAX_VOLUME
         # The minimum limit memory is equal to the memory allocated by default
         min_volume = settings.KUBERNETES_LIMITS_MIN_VOLUME
-        range_error = "Volume setting is not in allowed range: %sG~%sG" % (
-            min_volume, max_volume)
-        volume_size = int(data[:-1])
         if volume_size < min_volume or volume_size > max_volume:
-            raise serializers.ValidationError(range_error)
+            raise serializers.ValidationError(VOLUME_SIZE_MISMATCH_MSG)
         return data.upper()
 
+    validate_name = staticmethod(validate_name)
+
     @staticmethod
     def validate_path(data):
         logger.debug(f"mount validate_path data: {data}")
         new_data = {}
         for key, value in data.items():
-            if not re.match(PROCTYPE_MATCH, key):
-                raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+            validate_ptype(key)
             if value is None:  # use NoneType to unset an item
                 new_data[key] = value
                 continue
 
-            if not re.match(VOLUME_PATH, str(value)):
+            if not re.match(VOLUME_PATH_MATCH, str(value)):
                 raise serializers.ValidationError(
                     "Volume path format: /path")
             if value.endswith("/"):
@@ -604,11 +665,22 @@ def validate_path(data):
         logger.debug(f"mount validate_path new_data: {new_data}")
         return new_data
 
+    def validate_type(self, data):
+        if not re.match(VOLUME_TYPE_MATCH, data):
+            raise serializers.ValidationError(VOLUME_TYPE_MISMATCH_MSG)
+        elif data != "csi" and not self.initial_data.get("parameters", None):
+            raise serializers.ValidationError(
+                "parameters cannot be empty when the type is not csi.")
+        return data
+
+    @staticmethod
+    def validate_parameters(data):
+        return validate_json(data, VOLUMES_SCHEMA, serializers.ValidationError)
+
 
 class ResourceSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.resource.Resource` model."""
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     name = serializers.CharField(max_length=63, required=True)
     plan = serializers.CharField(max_length=128, required=True)
     data = JSONFieldSerializer(required=False, binary=True)
@@ -630,6 +702,8 @@ def update(self, instance, validated_data):
         instance.save()
         return instance
 
+    validate_name = staticmethod(validate_name)
+
 
 class MetricSerializer(serializers.Serializer):
     start = serializers.IntegerField(
@@ -638,12 +712,14 @@ class MetricSerializer(serializers.Serializer):
     stop = serializers.IntegerField(
         min_value=946656000, max_value=4102416000,
         required=False, default=lambda: int(time.time()))
+    step = serializers.IntegerField(
+        min_value=1, max_value=30, required=False, default=14)
     every = serializers.CharField(max_length=50, required=False, default='5m')
 
     def validate(self, attrs):
-        if not re.match(METRIC_EVERY, attrs["every"]):
+        if not re.match(METRIC_EVERY_MATCH, attrs["every"]):
             raise serializers.ValidationError(
-                "The format of every is:%s" % METRIC_EVERY.pattern
+                "The format of every is:%s" % METRIC_EVERY_MATCH.pattern
             )
         interval = attrs.get("stop") - attrs.get("start")
         if interval < 0 or interval > 3600 * 24:
@@ -660,7 +736,6 @@ def validate(self, attrs):
 
 class GatewaySerializer(serializers.Serializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
     name = serializers.CharField(max_length=63, required=True)
     listeners = serializers.JSONField(required=False)
     addresses = serializers.JSONField(read_only=True)
@@ -670,13 +745,7 @@ def to_representation(self, instance):
         representation['addresses'] = instance.addresses
         return representation
 
-    @staticmethod
-    def validate_port(value):
-        if not str(value).isnumeric():
-            raise serializers.ValidationError('port can only be a numeric value')
-        elif int(value) not in range(1, 65536):
-            raise serializers.ValidationError('port needs to be between 1 and 65535')
-        return value
+    validate_name = staticmethod(validate_name)
 
     @staticmethod
     def validate_protocol(value):
@@ -684,31 +753,23 @@ def validate_protocol(value):
             raise serializers.ValidationError(GATEWAY_PROTOCOL_MISMATCH_MSG)
         return value
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
-
-        return value
+    validate_port = staticmethod(validate_port)
+    validate_ptype = staticmethod(validate_ptype)
 
 
-class RouteSerializer(serializers.Serializer):
+class RouteSerializer(serializers.ModelSerializer):
     app = serializers.SlugRelatedField(slug_field='id', queryset=models.app.App.objects.all())
-    owner = serializers.ReadOnlyField(source='owner.username')
-    kind = serializers.CharField(max_length=15, required=False)
+    kind = serializers.CharField(max_length=15, required=True)
     name = serializers.CharField(max_length=63, required=True)
-    port = serializers.IntegerField()
-    procfile_type = serializers.CharField(max_length=63, required=True)
     rules = serializers.JSONField(required=False)
     parent_refs = serializers.JSONField(required=False)
 
-    @staticmethod
-    def validate_port(value):
-        if not str(value).isnumeric():
-            raise serializers.ValidationError('port can only be a numeric value')
-        elif int(value) not in range(1, 65536):
-            raise serializers.ValidationError('port needs to be between 1 and 65535')
-        return value
+    class Meta:
+        """Metadata options for a :class:`RouteSerializer`."""
+        model = models.gateway.Route
+        fields = '__all__'
+
+    validate_name = staticmethod(validate_name)
 
     @staticmethod
     def validate_kind(value):
@@ -716,19 +777,50 @@ def validate_kind(value):
             raise serializers.ValidationError(ROUTE_PROTOCOL_MISMATCH_MSG)
         return value
 
-    @staticmethod
-    def validate_procfile_type(value):
-        if not re.match(PROCTYPE_MATCH, value):
-            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+    def validate_rules(self, value):
+        kind = getattr(self, "initial_data", {}).get("kind", None)
+        if kind:
+            schema = getattr(rules, f"{kind.replace("Route", "")}_RULES_SCHEMA", "SCHEMA")
+        else:
+            schema = rules.SCHEMA
+        return validate_json(value, schema, serializers.ValidationError)
 
-        return value
 
-    @staticmethod
-    def validate_rules(value):
-        try:
-            jsonschema.validate(value, RULES_SCHEMA)
-        except jsonschema.ValidationError as e:
-            raise serializers.ValidationError(
-                "could not validate {}: {}".format(value, e.message)
-            )
-        return value
+class WorkspaceSerializer(serializers.ModelSerializer):
+    """Serialize Workspace model."""
+
+    class Meta:
+        model = models.workspace.Workspace
+        fields = '__all__'
+        read_only_fields = ['id', 'created', 'updated']
+        extra_kwargs = {
+            'name': {'validators': [models.workspace.validate_workspace_name]},
+        }
+
+    def update(self, instance, validated_data):
+        # Workspace name cannot be modified; the name field is read-only after creation.
+        validated_data.pop('name', None)
+        return super().update(instance, validated_data)
+
+
+class WorkspaceMemberSerializer(serializers.ModelSerializer):
+    """Serialize WorkspaceMember model."""
+    user = serializers.ReadOnlyField(source='user.username')
+    email = serializers.ReadOnlyField(source='user.email')
+    workspace = serializers.ReadOnlyField(source='workspace.name')
+
+    class Meta:
+        model = models.workspace.WorkspaceMember
+        fields = '__all__'
+        read_only_fields = ['id', 'created', 'updated']
+
+
+class WorkspaceInvitationSerializer(serializers.ModelSerializer):
+    """Serialize WorkspaceInvitation model."""
+    inviter = serializers.ReadOnlyField(source='inviter.username')
+    workspace = serializers.ReadOnlyField(source='workspace.name')
+
+    class Meta:
+        model = models.workspace.WorkspaceInvitation
+        fields = '__all__'
+        read_only_fields = ['id', 'token', 'created']
diff --git a/rootfs/api/serializers/schemas/__init__.py b/rootfs/api/serializers/schemas/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/rootfs/api/schemas/autoscale.py b/rootfs/api/serializers/schemas/autoscale.py
similarity index 100%
rename from rootfs/api/schemas/autoscale.py
rename to rootfs/api/serializers/schemas/autoscale.py
diff --git a/rootfs/api/serializers/schemas/config.py b/rootfs/api/serializers/schemas/config.py
new file mode 100644
index 000000000..4b15877d5
--- /dev/null
+++ b/rootfs/api/serializers/schemas/config.py
@@ -0,0 +1,30 @@
+SCHEMA = {
+    "description": "Config values are a list of config item.",
+    "$schema": "http://json-schema.org/schema#",
+    "items": {
+        "oneOf": [
+            {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "ptype": {"type": "string"},
+                    "value": {"oneOf": [{"type": "string"}, {"type": "null"}]},
+                },
+                "required": ["name", "ptype", "value"],
+                "additionalProperties": False,
+            },
+            {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "group": {"type": "string"},
+                    "value": {"oneOf": [{"type": "string"}, {"type": "null"}]},
+                },
+                "required": ["name", "group", "value"],
+                "additionalProperties": False,
+            },
+        ]
+    },
+    "minItems": 0,
+    "type": "array"
+}
diff --git a/rootfs/api/serializers/schemas/dryccfile.py b/rootfs/api/serializers/schemas/dryccfile.py
new file mode 100644
index 000000000..bb9729d2b
--- /dev/null
+++ b/rootfs/api/serializers/schemas/dryccfile.py
@@ -0,0 +1,85 @@
+
+FILENAME_REGEX = r'^(?P<name>[a-z0-9]+(\.[a-z0-9]+)*)$'
+PROCTYPE_REGEX = r'^(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)$'
+CONFIGKEY_REGEX = r'^[A-z0-9_\-\.]+$'
+CONFIGENV_SCHEMA = {
+    "type": "object",
+    "patternProperties": {
+        CONFIGKEY_REGEX: {"type": "string"},
+    },
+}
+SCHEMA = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "config": {
+            "type": "object",
+            "patternProperties": {
+                FILENAME_REGEX: CONFIGENV_SCHEMA,
+            },
+        },
+        "pipeline": {
+            "type": "object",
+            "patternProperties": {
+                FILENAME_REGEX: {
+                    "type": "object",
+                    "properties": {
+                        "kind": {"type": "string", "pattern": "^pipeline$"},
+                        "ptype": {"type": "string", "pattern": PROCTYPE_REGEX},
+                        "build": {
+                            "type": "object",
+                            "properties": {
+                                "docker": {"type": "string"},
+                                "arg": CONFIGENV_SCHEMA,
+                            },
+                            "additionalProperties": False,
+                        },
+                        "env": CONFIGENV_SCHEMA,
+                        "run": {
+                            "type": "object",
+                            "properties": {
+                                "command": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
+                                },
+                                "args": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
+                                },
+                                "image": {"type": "string"},
+                                "timeout": {"type": "integer"},
+                            },
+                            "additionalProperties": False,
+                        },
+                        "config": {
+                            "type": "array",
+                            "items": {
+                                "type": "string",
+                                "pattern": FILENAME_REGEX,
+                            },
+                        },
+                        "deploy": {
+                            "type": "object",
+                            "properties": {
+                                "command": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
+                                },
+                                "args": {
+                                    "type": "array",
+                                    "items": {"type": "string"},
+                                },
+                                "image": {"type": "string"},
+                            },
+                            "additionalProperties": False,
+                        }
+                    },
+                    "additionalProperties": False,
+                    "required": ["kind", "ptype", "deploy"],
+                },
+            },
+        },
+    },
+    "required": ["pipeline"],
+    "additionalProperties": False,
+}
diff --git a/rootfs/api/schemas/healthcheck.py b/rootfs/api/serializers/schemas/healthcheck.py
similarity index 58%
rename from rootfs/api/schemas/healthcheck.py
rename to rootfs/api/serializers/schemas/healthcheck.py
index 8c6aa2d07..c4cc604ca 100644
--- a/rootfs/api/schemas/healthcheck.py
+++ b/rootfs/api/serializers/schemas/healthcheck.py
@@ -3,8 +3,6 @@
 
     "type": "object",
     "properties": {
-        # Exec specifies the action to take.
-        # More info: http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_execaction
         "exec": {
             "type": "object",
             "properties": {
@@ -16,8 +14,14 @@
             },
             "required": ["command"]
         },
-        # HTTPGet specifies the http request to perform.
-        # More info: http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_httpgetaction
+        "grpc": {
+            "type": "object",
+            "properties": {
+                "port": {"type": "integer"},
+                "service": {"type": "string"},
+            },
+            "required": ["port"]
+        },
         "httpGet": {
             "type": "object",
             "properties": {
@@ -39,8 +43,6 @@
             },
             "required": ["port"]
         },
-        # TCPSocket specifies an action involving a TCP port.
-        # More info: http://kubernetes.io/docs/api-reference/v1/definitions/#_v1_tcpsocketaction
         "tcpSocket": {
             "type": "object",
             "properties": {
@@ -48,19 +50,16 @@
             },
             "required": ["port"]
         },
-        # Number of seconds after the container has started before liveness probes are initiated.
-        # More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
         "initialDelaySeconds": {"type": "integer"},
-        # Number of seconds after which the probe times out.
-        # More info: http://releases.k8s.io/HEAD/docs/user-guide/pod-states.md#container-probes
         "timeoutSeconds": {"type": "integer"},
-        # How often (in seconds) to perform the probe.
         "periodSeconds": {"type": "integer"},
-        # Minimum consecutive successes for the probe to be considered successful
-        # after having failed.
         "successThreshold": {"type": "integer"},
-        # Minimum consecutive failures for the probe to be considered
-        # failed after having succeeded.
         "failureThreshold": {"type": "integer"},
-    }
+    },
+    "oneOf": [
+        {"required": ["exec"]},
+        {"required": ["grpc"]},
+        {"required": ["httpGet"]},
+        {"required": ["tcpSocket"]},
+    ]
 }
diff --git a/rootfs/api/serializers/schemas/rules.py b/rootfs/api/serializers/schemas/rules.py
new file mode 100644
index 000000000..e12b6ae3c
--- /dev/null
+++ b/rootfs/api/serializers/schemas/rules.py
@@ -0,0 +1,2417 @@
+# flake8: noqa
+
+
+TCP_RULES_SCHEMA = {
+    "description": "Rules are a list of TCP matchers and actions.",
+    "items": {
+        "description": "TCPRouteRule is the configuration for a given rule.",
+        "properties": {
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent. If unspecified or invalid (refers to a non-existent resource or a\nService with no endpoints), the underlying implementation MUST actively\nreject connection attempts to this backend. Connection rejections must\nrespect weight; if an invalid backend is requested to have 80% of\nconnections, then 80% of connections must be rejected instead.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Extended for Kubernetes ServiceImport\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Extended",
+                "items": {
+                    "description": "BackendRef defines how a Route should forward a request to a Kubernetes\nresource.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>\n\n\nNote that when the BackendTLSPolicy object is enabled by the implementation,\nthere are some extra rules about validity to consider here. See the fields\nwhere this struct is used for more information about the exact behavior.",
+                    "properties": {
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "minItems": 1,
+                "type": "array"
+            }
+        },
+        "type": "object"
+    },
+    "maxItems": 16,
+    "minItems": 1,
+    "type": "array"
+}
+
+UDP_RULES_SCHEMA = {
+    "description": "Rules are a list of UDP matchers and actions.",
+    "items": {
+        "description": "UDPRouteRule is the configuration for a given rule.",
+        "properties": {
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent. If unspecified or invalid (refers to a non-existent resource or a\nService with no endpoints), the underlying implementation MUST actively\nreject connection attempts to this backend. Packet drops must\nrespect weight; if an invalid backend is requested to have 80% of\nthe packets, then 80% of packets must be dropped instead.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Extended for Kubernetes ServiceImport\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Extended",
+                "items": {
+                    "description": "BackendRef defines how a Route should forward a request to a Kubernetes\nresource.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>\n\n\nNote that when the BackendTLSPolicy object is enabled by the implementation,\nthere are some extra rules about validity to consider here. See the fields\nwhere this struct is used for more information about the exact behavior.",
+                    "properties": {
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "minItems": 1,
+                "type": "array"
+            }
+        },
+        "type": "object"
+    },
+    "maxItems": 16,
+    "minItems": 1,
+    "type": "array"
+}
+
+TLS_RULES_SCHEMA = {
+    "description": "Rules are a list of TLS matchers and actions.",
+    "items": {
+        "description": "TLSRouteRule is the configuration for a given rule.",
+        "properties": {
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent. If unspecified or invalid (refers to a non-existent resource or\na Service with no endpoints), the rule performs no forwarding; if no\nfilters are specified that would result in a response being sent, the\nunderlying implementation must actively reject request attempts to this\nbackend, by rejecting the connection or returning a 500 status code.\nRequest rejections must respect weight; if an invalid backend is\nrequested to have 80% of requests, then 80% of requests must be rejected\ninstead.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Extended for Kubernetes ServiceImport\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Extended",
+                "items": {
+                    "description": "BackendRef defines how a Route should forward a request to a Kubernetes\nresource.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>\n\n\nNote that when the BackendTLSPolicy object is enabled by the implementation,\nthere are some extra rules about validity to consider here. See the fields\nwhere this struct is used for more information about the exact behavior.",
+                    "properties": {
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "minItems": 1,
+                "type": "array"
+            }
+        },
+        "type": "object"
+    },
+    "maxItems": 16,
+    "minItems": 1,
+    "type": "array"
+}
+
+HTTP_RULES_SCHEMA = {
+    "description": "Rules are a list of HTTP matchers, filters and actions.",
+    "items": {
+        "properties": {
+            "backendRefs": {
+                "items": {
+                    "properties": {
+                        "filters": {
+                            "items": {
+                                "properties": {
+                                    "extensionRef": {
+                                        "properties": {
+                                            "group": {
+                                                "description": "",
+                                                "maxLength": 253,
+                                                "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "kind": {
+                                                "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                                "maxLength": 63,
+                                                "minLength": 1,
+                                                "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                "type": "string"
+                                            },
+                                            "name": {
+                                                "description": "Name is the name of the referent.",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "group",
+                                            "kind",
+                                            "name"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "requestHeaderModifier": {
+                                        "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "requestMirror": {
+                                        "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "backendRef": {
+                                                "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                                "properties": {
+                                                    "group": {
+                                                        "default": "",
+                                                        "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                                        "maxLength": 253,
+                                                        "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                        "type": "string"
+                                                    },
+                                                    "kind": {
+                                                        "default": "Service",
+                                                        "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "name": {
+                                                        "description": "Name is the name of the referent.",
+                                                        "maxLength": 253,
+                                                        "minLength": 1,
+                                                        "type": "string"
+                                                    },
+                                                    "namespace": {
+                                                        "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "port": {
+                                                        "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                                        "format": "int32",
+                                                        "maximum": 65535,
+                                                        "minimum": 1,
+                                                        "type": "integer"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "name"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "Must have port for Service reference",
+                                                        "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                                    }
+                                                ]
+                                            }
+                                        },
+                                        "required": [
+                                            "backendRef"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "requestRedirect": {
+                                        "description": "RequestRedirect defines a schema for a filter that responds to the\nrequest with an HTTP redirection.\n\n\nSupport: Core",
+                                        "properties": {
+                                            "hostname": {
+                                                "description": "Hostname is the hostname to be used in the value of the `Location`\nheader in the response.\nWhen empty, the hostname in the `Host` header of the request is used.\n\n\nSupport: Core",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "path": {
+                                                "description": "Path defines parameters used to modify the path of the incoming request.\nThe modified path is then used to construct the `Location` header. When\nempty, the request path is used as-is.\n\n\nSupport: Extended",
+                                                "properties": {
+                                                    "replaceFullPath": {
+                                                        "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "replacePrefixMatch": {
+                                                        "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "type": {
+                                                        "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                                        "enum": [
+                                                            "ReplaceFullPath",
+                                                            "ReplacePrefixMatch"
+                                                        ],
+                                                        "type": "string"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "type"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                                        "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                                        "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                                    },
+                                                    {
+                                                        "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                                        "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                                        "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                                    }
+                                                ]
+                                            },
+                                            "port": {
+                                                "description": "Port is the port to be used in the value of the `Location`\nheader in the response.\n\n\nIf no port is specified, the redirect port MUST be derived using the\nfollowing rules:\n\n\n* If redirect scheme is not-empty, the redirect port MUST be the well-known\n  port associated with the redirect scheme. Specifically \"http\" to port 80\n  and \"https\" to port 443. If the redirect scheme does not have a\n  well-known port, the listener port of the Gateway SHOULD be used.\n* If redirect scheme is empty, the redirect port MUST be the Gateway\n  Listener port.\n\n\nImplementations SHOULD NOT add the port number in the 'Location'\nheader in the following cases:\n\n\n* A Location header that will use HTTP (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 80.\n* A Location header that will use HTTPS (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 443.\n\n\nSupport: Extended",
+                                                "format": "int32",
+                                                "maximum": 65535,
+                                                "minimum": 1,
+                                                "type": "integer"
+                                            },
+                                            "scheme": {
+                                                "description": "Scheme is the scheme to be used in the value of the `Location` header in\nthe response. When empty, the scheme of the request is used.\n\n\nScheme redirects can affect the port of the redirect, for more information,\nrefer to the documentation for the port field of this filter.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Extended",
+                                                "enum": [
+                                                    "http",
+                                                    "https"
+                                                ],
+                                                "type": "string"
+                                            },
+                                            "statusCode": {
+                                                "default": 302,
+                                                "description": "StatusCode is the HTTP status code to be used in response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Core",
+                                                "enum": [
+                                                    301,
+                                                    302
+                                                ],
+                                                "type": "integer"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "responseHeaderModifier": {
+                                        "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "type": {
+                                        "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations must support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by\n  specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` should be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                        "enum": [
+                                            "RequestHeaderModifier",
+                                            "ResponseHeaderModifier",
+                                            "RequestMirror",
+                                            "RequestRedirect",
+                                            "URLRewrite",
+                                            "ExtensionRef"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "urlRewrite": {
+                                        "description": "URLRewrite defines a schema for a filter that modifies a request during forwarding.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "hostname": {
+                                                "description": "Hostname is the value to be used to replace the Host header value during\nforwarding.\n\n\nSupport: Extended",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "path": {
+                                                "description": "Path defines a path rewrite.\n\n\nSupport: Extended",
+                                                "properties": {
+                                                    "replaceFullPath": {
+                                                        "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "replacePrefixMatch": {
+                                                        "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                                        "maxLength": 1024,
+                                                        "type": "string"
+                                                    },
+                                                    "type": {
+                                                        "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                                        "enum": [
+                                                            "ReplaceFullPath",
+                                                            "ReplacePrefixMatch"
+                                                        ],
+                                                        "type": "string"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "type"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                                        "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                                        "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                                    },
+                                                    {
+                                                        "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                                        "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                                    },
+                                                    {
+                                                        "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                                        "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                                    }
+                                                ]
+                                            }
+                                        },
+                                        "type": "object"
+                                    }
+                                },
+                                "required": [
+                                    "type"
+                                ],
+                                "type": "object",
+                                "x-kubernetes-validations": [
+                                    {
+                                        "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                                        "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                                        "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                                        "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                                        "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                                        "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                                        "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.requestRedirect must be nil if the filter.type is not RequestRedirect",
+                                        "rule": "!(has(self.requestRedirect) && self.type != 'RequestRedirect')"
+                                    },
+                                    {
+                                        "message": "filter.requestRedirect must be specified for RequestRedirect filter.type",
+                                        "rule": "!(!has(self.requestRedirect) && self.type == 'RequestRedirect')"
+                                    },
+                                    {
+                                        "message": "filter.urlRewrite must be nil if the filter.type is not URLRewrite",
+                                        "rule": "!(has(self.urlRewrite) && self.type != 'URLRewrite')"
+                                    },
+                                    {
+                                        "message": "filter.urlRewrite must be specified for URLRewrite filter.type",
+                                        "rule": "!(!has(self.urlRewrite) && self.type == 'URLRewrite')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                                        "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                                        "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                                    }
+                                ]
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both",
+                                    "rule": "!(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite'))"
+                                },
+                                {
+                                    "message": "May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both",
+                                    "rule": "!(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite'))"
+                                },
+                                {
+                                    "message": "RequestHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                                },
+                                {
+                                    "message": "ResponseHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                                },
+                                {
+                                    "message": "RequestRedirect filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'RequestRedirect').size() <= 1"
+                                },
+                                {
+                                    "message": "URLRewrite filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'URLRewrite').size() <= 1"
+                                }
+                            ]
+                        },
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "type": "array"
+            },
+            "filters": {
+                "description": "Filters define the filters that are applied to requests that match\nthis rule.\n\n\nWherever possible, implementations SHOULD implement filters in the order\nthey are specified.\n\n\nImplementations MAY choose to implement this ordering strictly, rejecting\nany combination or order of filters that can not be supported. If implementations\nchoose a strict interpretation of filter ordering, they MUST clearly document\nthat behavior.\n\n\nTo reject an invalid combination or order of filters, implementations SHOULD\nconsider the Route Rules with this configuration invalid. If all Route Rules\nin a Route are invalid, the entire Route would be considered invalid. If only\na portion of Route Rules are invalid, implementations MUST set the\n\"PartiallyInvalid\" condition for the Route.\n\n\nConformance-levels at this level are defined based on the type of filter:\n\n\n- ALL core filters MUST be supported by all implementations.\n- Implementers are encouraged to support extended filters.\n- Implementation-specific custom filters have no API guarantees across\n  implementations.\n\n\nSpecifying the same filter multiple times is not supported unless explicitly\nindicated in the filter.\n\n\nAll filters are expected to be compatible with each other except for the\nURLRewrite and RequestRedirect filters, which may not be combined. If an\nimplementation can not support other combinations of filters, they must clearly\ndocument that limitation. In cases where incompatible or unsupported\nfilters are specified and cause the `Accepted` condition to be set to status\n`False`, implementations may use the `IncompatibleFilters` reason to specify\nthis configuration error.\n\n\nSupport: Core",
+                "items": {
+                    "description": "HTTPRouteFilter defines processing steps that must be completed during the\nrequest or response lifecycle. HTTPRouteFilters are meant as an extension\npoint to express processing that may be done in Gateway implementations. Some\nexamples include request or response modification, implementing\nauthentication strategies, rate-limiting, and traffic shaping. API\nguarantee/conformance is defined based on the type of the filter.",
+                    "properties": {
+                        "extensionRef": {
+                            "description": "ExtensionRef is an optional, implementation-specific extension to the\n\"filter\" behavior.  For example, resource \"myroutefilter\" in group\n\"networking.example.net\"). ExtensionRef MUST NOT be used for core and\nextended filters.\n\n\nThis filter can be used multiple times within the same rule.\n\n\nSupport: Implementation-specific",
+                            "properties": {
+                                "group": {
+                                    "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                    "maxLength": 253,
+                                    "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "kind": {
+                                    "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                    "maxLength": 63,
+                                    "minLength": 1,
+                                    "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                    "type": "string"
+                                },
+                                "name": {
+                                    "description": "Name is the name of the referent.",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "group",
+                                "kind",
+                                "name"
+                            ],
+                            "type": "object"
+                        },
+                        "requestHeaderModifier": {
+                            "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "requestMirror": {
+                            "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                            "properties": {
+                                "backendRef": {
+                                    "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                    "properties": {
+                                        "group": {
+                                            "default": "",
+                                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                            "maxLength": 253,
+                                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                            "type": "string"
+                                        },
+                                        "kind": {
+                                            "default": "Service",
+                                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "name": {
+                                            "description": "Name is the name of the referent.",
+                                            "maxLength": 253,
+                                            "minLength": 1,
+                                            "type": "string"
+                                        },
+                                        "namespace": {
+                                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "port": {
+                                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                            "format": "int32",
+                                            "maximum": 65535,
+                                            "minimum": 1,
+                                            "type": "integer"
+                                        }
+                                    },
+                                    "required": [
+                                        "name"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "Must have port for Service reference",
+                                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                        }
+                                    ]
+                                }
+                            },
+                            "required": [
+                                "backendRef"
+                            ],
+                            "type": "object"
+                        },
+                        "requestRedirect": {
+                            "description": "RequestRedirect defines a schema for a filter that responds to the\nrequest with an HTTP redirection.\n\n\nSupport: Core",
+                            "properties": {
+                                "hostname": {
+                                    "description": "Hostname is the hostname to be used in the value of the `Location`\nheader in the response.\nWhen empty, the hostname in the `Host` header of the request is used.\n\n\nSupport: Core",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "path": {
+                                    "description": "Path defines parameters used to modify the path of the incoming request.\nThe modified path is then used to construct the `Location` header. When\nempty, the request path is used as-is.\n\n\nSupport: Extended",
+                                    "properties": {
+                                        "replaceFullPath": {
+                                            "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "replacePrefixMatch": {
+                                            "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "type": {
+                                            "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                            "enum": [
+                                                "ReplaceFullPath",
+                                                "ReplacePrefixMatch"
+                                            ],
+                                            "type": "string"
+                                        }
+                                    },
+                                    "required": [
+                                        "type"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                            "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                            "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                        },
+                                        {
+                                            "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                            "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                            "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                        }
+                                    ]
+                                },
+                                "port": {
+                                    "description": "Port is the port to be used in the value of the `Location`\nheader in the response.\n\n\nIf no port is specified, the redirect port MUST be derived using the\nfollowing rules:\n\n\n* If redirect scheme is not-empty, the redirect port MUST be the well-known\n  port associated with the redirect scheme. Specifically \"http\" to port 80\n  and \"https\" to port 443. If the redirect scheme does not have a\n  well-known port, the listener port of the Gateway SHOULD be used.\n* If redirect scheme is empty, the redirect port MUST be the Gateway\n  Listener port.\n\n\nImplementations SHOULD NOT add the port number in the 'Location'\nheader in the following cases:\n\n\n* A Location header that will use HTTP (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 80.\n* A Location header that will use HTTPS (whether that is determined via\n  the Listener protocol or the Scheme field) _and_ use port 443.\n\n\nSupport: Extended",
+                                    "format": "int32",
+                                    "maximum": 65535,
+                                    "minimum": 1,
+                                    "type": "integer"
+                                },
+                                "scheme": {
+                                    "description": "Scheme is the scheme to be used in the value of the `Location` header in\nthe response. When empty, the scheme of the request is used.\n\n\nScheme redirects can affect the port of the redirect, for more information,\nrefer to the documentation for the port field of this filter.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Extended",
+                                    "enum": [
+                                        "http",
+                                        "https"
+                                    ],
+                                    "type": "string"
+                                },
+                                "statusCode": {
+                                    "default": 302,
+                                    "description": "StatusCode is the HTTP status code to be used in response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.\n\n\nSupport: Core",
+                                    "enum": [
+                                        301,
+                                        302
+                                    ],
+                                    "type": "integer"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "responseHeaderModifier": {
+                            "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "type": {
+                            "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations must support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by\n  specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` should be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                            "enum": [
+                                "RequestHeaderModifier",
+                                "ResponseHeaderModifier",
+                                "RequestMirror",
+                                "RequestRedirect",
+                                "URLRewrite",
+                                "ExtensionRef"
+                            ],
+                            "type": "string"
+                        },
+                        "urlRewrite": {
+                            "description": "URLRewrite defines a schema for a filter that modifies a request during forwarding.\n\n\nSupport: Extended",
+                            "properties": {
+                                "hostname": {
+                                    "description": "Hostname is the value to be used to replace the Host header value during\nforwarding.\n\n\nSupport: Extended",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "path": {
+                                    "description": "Path defines a path rewrite.\n\n\nSupport: Extended",
+                                    "properties": {
+                                        "replaceFullPath": {
+                                            "description": "ReplaceFullPath specifies the value with which to replace the full path\nof a request during a rewrite or redirect.",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "replacePrefixMatch": {
+                                            "description": "ReplacePrefixMatch specifies the value with which to replace the prefix\nmatch of a request during a rewrite or redirect. For example, a request\nto \"/foo/bar\" with a prefix match of \"/foo\" and a ReplacePrefixMatch\nof \"/xyz\" would be modified to \"/xyz/bar\".\n\n\nNote that this matches the behavior of the PathPrefix match type. This\nmatches full path elements. A path element refers to the list of labels\nin the path split by the `/` separator. When specified, a trailing `/` is\nignored. For example, the paths `/abc`, `/abc/`, and `/abc/def` would all\nmatch the prefix `/abc`, but the path `/abcd` would not.\n\n\nReplacePrefixMatch is only compatible with a `PathPrefix` HTTPRouteMatch.\nUsing any other HTTPRouteMatch type on the same HTTPRouteRule will result in\nthe implementation setting the Accepted Condition for the Route to `status: False`.\n\n\nRequest Path | Prefix Match | Replace Prefix | Modified Path\n-------------|--------------|----------------|----------\n/foo/bar     | /foo         | /xyz           | /xyz/bar\n/foo/bar     | /foo         | /xyz/          | /xyz/bar\n/foo/bar     | /foo/        | /xyz           | /xyz/bar\n/foo/bar     | /foo/        | /xyz/          | /xyz/bar\n/foo         | /foo         | /xyz           | /xyz\n/foo/        | /foo         | /xyz           | /xyz/\n/foo/bar     | /foo         | <empty string> | /bar\n/foo/        | /foo         | <empty string> | /\n/foo         | /foo         | <empty string> | /\n/foo/        | /foo         | /              | /\n/foo         | /foo         | /              | /",
+                                            "maxLength": 1024,
+                                            "type": "string"
+                                        },
+                                        "type": {
+                                            "description": "Type defines the type of path modifier. Additional types may be\nadded in a future release of the API.\n\n\nNote that values may be added to this enum, implementations\nmust ensure that unknown values will not cause a crash.\n\n\nUnknown values here must result in the implementation setting the\nAccepted Condition for the Route to `status: False`, with a\nReason of `UnsupportedValue`.",
+                                            "enum": [
+                                                "ReplaceFullPath",
+                                                "ReplacePrefixMatch"
+                                            ],
+                                            "type": "string"
+                                        }
+                                    },
+                                    "required": [
+                                        "type"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "replaceFullPath must be specified when type is set to 'ReplaceFullPath'",
+                                            "rule": "self.type == 'ReplaceFullPath' ? has(self.replaceFullPath) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplaceFullPath' when replaceFullPath is set",
+                                            "rule": "has(self.replaceFullPath) ? self.type == 'ReplaceFullPath' : true"
+                                        },
+                                        {
+                                            "message": "replacePrefixMatch must be specified when type is set to 'ReplacePrefixMatch'",
+                                            "rule": "self.type == 'ReplacePrefixMatch' ? has(self.replacePrefixMatch) : true"
+                                        },
+                                        {
+                                            "message": "type must be 'ReplacePrefixMatch' when replacePrefixMatch is set",
+                                            "rule": "has(self.replacePrefixMatch) ? self.type == 'ReplacePrefixMatch' : true"
+                                        }
+                                    ]
+                                }
+                            },
+                            "type": "object"
+                        }
+                    },
+                    "required": [
+                        "type"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                            "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                            "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                            "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                            "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                            "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                            "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.requestRedirect must be nil if the filter.type is not RequestRedirect",
+                            "rule": "!(has(self.requestRedirect) && self.type != 'RequestRedirect')"
+                        },
+                        {
+                            "message": "filter.requestRedirect must be specified for RequestRedirect filter.type",
+                            "rule": "!(!has(self.requestRedirect) && self.type == 'RequestRedirect')"
+                        },
+                        {
+                            "message": "filter.urlRewrite must be nil if the filter.type is not URLRewrite",
+                            "rule": "!(has(self.urlRewrite) && self.type != 'URLRewrite')"
+                        },
+                        {
+                            "message": "filter.urlRewrite must be specified for URLRewrite filter.type",
+                            "rule": "!(!has(self.urlRewrite) && self.type == 'URLRewrite')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                            "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                            "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "type": "array",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "May specify either httpRouteFilterRequestRedirect or httpRouteFilterRequestRewrite, but not both",
+                        "rule": "!(self.exists(f, f.type == 'RequestRedirect') && self.exists(f, f.type == 'URLRewrite'))"
+                    },
+                    {
+                        "message": "RequestHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                    },
+                    {
+                        "message": "ResponseHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                    },
+                    {
+                        "message": "RequestRedirect filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'RequestRedirect').size() <= 1"
+                    },
+                    {
+                        "message": "URLRewrite filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'URLRewrite').size() <= 1"
+                    }
+                ]
+            },
+            "matches": {
+                "default": [
+                    {
+                        "path": {
+                            "type": "PathPrefix",
+                            "value": "/"
+                        }
+                    }
+                ],
+                "description": "Matches define conditions used for matching the rule against incoming\nHTTP requests. Each match is independent, i.e. this rule will be matched\nif **any** one of the matches is satisfied.\n\n\nFor example, take the following matches configuration:\n\n\n```\nmatches:\n- path:\n    value: \"/foo\"\n  headers:\n  - name: \"version\"\n    value: \"v2\"\n- path:\n    value: \"/v2/foo\"\n```\n\n\nFor a request to match against this rule, a request must satisfy\nEITHER of the two conditions:\n\n\n- path prefixed with `/foo` AND contains the header `version: v2`\n- path prefix of `/v2/foo`\n\n\nSee the documentation for HTTPRouteMatch on how to specify multiple\nmatch conditions that should be ANDed together.\n\n\nIf no matches are specified, the default is a prefix\npath match on \"/\", which has the effect of matching every\nHTTP request.\n\n\nProxy or Load Balancer routing configuration generated from HTTPRoutes\nMUST prioritize matches based on the following criteria, continuing on\nties. Across all rules specified on applicable Routes, precedence must be\ngiven to the match having:\n\n\n* \"Exact\" path match.\n* \"Prefix\" path match with largest number of characters.\n* Method match.\n* Largest number of header matches.\n* Largest number of query param matches.\n\n\nNote: The precedence of RegularExpression path matches are implementation-specific.\n\n\nIf ties still exist across multiple Routes, matching precedence MUST be\ndetermined in order of the following criteria, continuing on ties:\n\n\n* The oldest Route based on creation timestamp.\n* The Route appearing first in alphabetical order by\n  \"{namespace}/{name}\".\n\n\nIf ties still exist within an HTTPRoute, matching precedence MUST be granted\nto the FIRST matching rule (in list order) with a match meeting the above\ncriteria.\n\n\nWhen no rules matching a request have been successfully attached to the\nparent a request is coming from, a HTTP 404 status code MUST be returned.",
+                "items": {
+                    "description": "HTTPRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\n\nFor example, the match below will match a HTTP request only if its path\nstarts with `/foo` AND it contains the `version: v1` header:\n\n\n```\nmatch:\n\n\n\tpath:\n\t  value: \"/foo\"\n\theaders:\n\t- name: \"version\"\n\t  value \"v1\"\n\n\n```",
+                    "properties": {
+                        "headers": {
+                            "description": "Headers specifies HTTP request header matchers. Multiple match values are\nANDed together, meaning, a request must match all the specified headers\nto select the route.",
+                            "items": {
+                                "description": "HTTPHeaderMatch describes how to select a HTTP route by matching HTTP request\nheaders.",
+                                "properties": {
+                                    "name": {
+                                        "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, only the first\nentry with an equivalent name MUST be considered for a match. Subsequent\nentries with an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.\n\n\nWhen a header is repeated in an HTTP request, it is\nimplementation-specific behavior as to how this is represented.\nGenerally, proxies should follow the guidance from the RFC:\nhttps://www.rfc-editor.org/rfc/rfc7230.html#section-3.2.2 regarding\nprocessing a repeated header, with special handling for \"Set-Cookie\".",
+                                        "maxLength": 256,
+                                        "minLength": 1,
+                                        "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                        "type": "string"
+                                    },
+                                    "type": {
+                                        "default": "Exact",
+                                        "description": "Type specifies how to match against the value of the header.\n\n\nSupport: Core (Exact)\n\n\nSupport: Implementation-specific (RegularExpression)\n\n\nSince RegularExpression HeaderMatchType has implementation-specific\nconformance, implementations can support POSIX, PCRE or any other dialects\nof regular expressions. Please read the implementation's documentation to\ndetermine the supported dialect.",
+                                        "enum": [
+                                            "Exact",
+                                            "RegularExpression"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "value": {
+                                        "description": "Value is the value of HTTP Header to be matched.",
+                                        "maxLength": 4096,
+                                        "minLength": 1,
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "name",
+                                    "value"
+                                ],
+                                "type": "object"
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-list-map-keys": [
+                                "name"
+                            ],
+                            "x-kubernetes-list-type": "map"
+                        },
+                        "method": {
+                            "description": "Method specifies HTTP method matcher.\nWhen specified, this route will be matched only if the request has the\nspecified method.\n\n\nSupport: Extended",
+                            "enum": [
+                                "GET",
+                                "HEAD",
+                                "POST",
+                                "PUT",
+                                "DELETE",
+                                "CONNECT",
+                                "OPTIONS",
+                                "TRACE",
+                                "PATCH"
+                            ],
+                            "type": "string"
+                        },
+                        "path": {
+                            "default": {
+                                "type": "PathPrefix",
+                                "value": "/"
+                            },
+                            "description": "Path specifies a HTTP request path matcher. If this field is not\nspecified, a default prefix match on the \"/\" path is provided.",
+                            "properties": {
+                                "type": {
+                                    "default": "PathPrefix",
+                                    "description": "Type specifies how to match against the path Value.\n\n\nSupport: Core (Exact, PathPrefix)\n\n\nSupport: Implementation-specific (RegularExpression)",
+                                    "enum": [
+                                        "Exact",
+                                        "PathPrefix",
+                                        "RegularExpression"
+                                    ],
+                                    "type": "string"
+                                },
+                                "value": {
+                                    "default": "/",
+                                    "description": "Value of the HTTP path to match against.",
+                                    "maxLength": 1024,
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "value must be an absolute path and start with '/' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? self.value.startsWith('/') : true"
+                                },
+                                {
+                                    "message": "must not contain '//' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('//') : true"
+                                },
+                                {
+                                    "message": "must not contain '/./' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('/./') : true"
+                                },
+                                {
+                                    "message": "must not contain '/../' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('/../') : true"
+                                },
+                                {
+                                    "message": "must not contain '%2f' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('%2f') : true"
+                                },
+                                {
+                                    "message": "must not contain '%2F' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('%2F') : true"
+                                },
+                                {
+                                    "message": "must not contain '#' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.contains('#') : true"
+                                },
+                                {
+                                    "message": "must not end with '/..' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.endsWith('/..') : true"
+                                },
+                                {
+                                    "message": "must not end with '/.' when type one of ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? !self.value.endsWith('/.') : true"
+                                },
+                                {
+                                    "message": "type must be one of ['Exact', 'PathPrefix', 'RegularExpression']",
+                                    "rule": "self.type in ['Exact','PathPrefix'] || self.type == 'RegularExpression'"
+                                },
+                                {
+                                    "message": "must only contain valid characters (matching ^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$) for types ['Exact', 'PathPrefix']",
+                                    "rule": "(self.type in ['Exact','PathPrefix']) ? self.value.matches(r\"\"\"^(?:[-A-Za-z0-9/._~!$&'()*+,;=:@]|[%][0-9a-fA-F]{2})+$\"\"\") : true"
+                                }
+                            ]
+                        },
+                        "queryParams": {
+                            "description": "QueryParams specifies HTTP query parameter matchers. Multiple match\nvalues are ANDed together, meaning, a request must match all the\nspecified query parameters to select the route.\n\n\nSupport: Extended",
+                            "items": {
+                                "description": "HTTPQueryParamMatch describes how to select a HTTP route by matching HTTP\nquery parameters.",
+                                "properties": {
+                                    "name": {
+                                        "description": "Name is the name of the HTTP query param to be matched. This must be an\nexact string match. (See\nhttps://tools.ietf.org/html/rfc7230#section-2.7.3).\n\n\nIf multiple entries specify equivalent query param names, only the first\nentry with an equivalent name MUST be considered for a match. Subsequent\nentries with an equivalent query param name MUST be ignored.\n\n\nIf a query param is repeated in an HTTP request, the behavior is\npurposely left undefined, since different data planes have different\ncapabilities. However, it is *recommended* that implementations should\nmatch against the first value of the param if the data plane supports it,\nas this behavior is expected in other load balancing contexts outside of\nthe Gateway API.\n\n\nUsers SHOULD NOT route traffic based on repeated query params to guard\nthemselves against potential differences in the implementations.",
+                                        "maxLength": 256,
+                                        "minLength": 1,
+                                        "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                        "type": "string"
+                                    },
+                                    "type": {
+                                        "default": "Exact",
+                                        "description": "Type specifies how to match against the value of the query parameter.\n\n\nSupport: Extended (Exact)\n\n\nSupport: Implementation-specific (RegularExpression)\n\n\nSince RegularExpression QueryParamMatchType has Implementation-specific\nconformance, implementations can support POSIX, PCRE or any other\ndialects of regular expressions. Please read the implementation's\ndocumentation to determine the supported dialect.",
+                                        "enum": [
+                                            "Exact",
+                                            "RegularExpression"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "value": {
+                                        "description": "Value is the value of HTTP query param to be matched.",
+                                        "maxLength": 1024,
+                                        "minLength": 1,
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "name",
+                                    "value"
+                                ],
+                                "type": "object"
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-list-map-keys": [
+                                "name"
+                            ],
+                            "x-kubernetes-list-type": "map"
+                        }
+                    },
+                    "type": "object"
+                },
+                "maxItems": 8,
+                "type": "array"
+            },
+            "sessionPersistence": {
+                "description": "SessionPersistence defines and configures session persistence\nfor the route rule.\n\n\nSupport: Extended\n\n\n",
+                "properties": {
+                    "absoluteTimeout": {
+                        "description": "AbsoluteTimeout defines the absolute timeout of the persistent\nsession. Once the AbsoluteTimeout duration has elapsed, the\nsession becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "cookieConfig": {
+                        "description": "CookieConfig provides configuration settings that are specific\nto cookie-based session persistence.\n\n\nSupport: Core",
+                        "properties": {
+                            "lifetimeType": {
+                                "default": "Session",
+                                "description": "LifetimeType specifies whether the cookie has a permanent or\nsession-based lifetime. A permanent cookie persists until its\nspecified expiry time, defined by the Expires or Max-Age cookie\nattributes, while a session cookie is deleted when the current\nsession ends.\n\n\nWhen set to \"Permanent\", AbsoluteTimeout indicates the\ncookie's lifetime via the Expires or Max-Age cookie attributes\nand is required.\n\n\nWhen set to \"Session\", AbsoluteTimeout indicates the\nabsolute lifetime of the cookie tracked by the gateway and\nis optional.\n\n\nSupport: Core for \"Session\" type\n\n\nSupport: Extended for \"Permanent\" type",
+                                "enum": [
+                                    "Permanent",
+                                    "Session"
+                                ],
+                                "type": "string"
+                            }
+                        },
+                        "type": "object"
+                    },
+                    "idleTimeout": {
+                        "description": "IdleTimeout defines the idle timeout of the persistent session.\nOnce the session has been idle for more than the specified\nIdleTimeout duration, the session becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "sessionName": {
+                        "description": "SessionName defines the name of the persistent session token\nwhich may be reflected in the cookie or the header. Users\nshould avoid reusing session names to prevent unintended\nconsequences, such as rejection or unpredictable behavior.\n\n\nSupport: Implementation-specific",
+                        "maxLength": 128,
+                        "type": "string"
+                    },
+                    "type": {
+                        "default": "Cookie",
+                        "description": "Type defines the type of session persistence such as through\nthe use a header or cookie. Defaults to cookie based session\npersistence.\n\n\nSupport: Core for \"Cookie\" type\n\n\nSupport: Extended for \"Header\" type",
+                        "enum": [
+                            "Cookie",
+                            "Header"
+                        ],
+                        "type": "string"
+                    }
+                },
+                "type": "object",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "AbsoluteTimeout must be specified when cookie lifetimeType is Permanent",
+                        "rule": "!has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != 'Permanent' || has(self.absoluteTimeout)"
+                    }
+                ]
+            },
+            "timeouts": {
+                "description": "Timeouts defines the timeouts that can be configured for an HTTP request.\n\n\nSupport: Extended\n\n\n",
+                "properties": {
+                    "backendRequest": {
+                        "description": "BackendRequest specifies a timeout for an individual request from the gateway\nto a backend. This covers the time from when the request first starts being\nsent from the gateway to when the full response has been received from the backend.\n\n\nSetting a timeout to the zero duration (e.g. \"0s\") SHOULD disable the timeout\ncompletely. Implementations that cannot completely disable the timeout MUST\ninstead interpret the zero duration as the longest possible value to which\nthe timeout can be set.\n\n\nAn entire client HTTP transaction with a gateway, covered by the Request timeout,\nmay result in more than one call from the gateway to the destination backend,\nfor example, if automatic retries are supported.\n\n\nBecause the Request timeout encompasses the BackendRequest timeout, the value of\nBackendRequest must be <= the value of Request timeout.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "request": {
+                        "description": "Request specifies the maximum duration for a gateway to respond to an HTTP request.\nIf the gateway has not been able to respond before this deadline is met, the gateway\nMUST return a timeout error.\n\n\nFor example, setting the `rules.timeouts.request` field to the value `10s` in an\n`HTTPRoute` will cause a timeout if a client request is taking longer than 10 seconds\nto complete.\n\n\nSetting a timeout to the zero duration (e.g. \"0s\") SHOULD disable the timeout\ncompletely. Implementations that cannot completely disable the timeout MUST\ninstead interpret the zero duration as the longest possible value to which\nthe timeout can be set.\n\n\nThis timeout is intended to cover as close to the whole request-response transaction\nas possible although an implementation MAY choose to start the timeout after the entire\nrequest stream has been received instead of immediately after the transaction is\ninitiated by the client.\n\n\nWhen this field is unspecified, request timeout behavior is implementation-specific.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    }
+                },
+                "type": "object",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "backendRequest timeout cannot be longer than request timeout",
+                        "rule": "!(has(self.request) && has(self.backendRequest) && duration(self.request) != duration('0s') && duration(self.backendRequest) > duration(self.request))"
+                    }
+                ]
+            }
+        },
+        "type": "object",
+        "x-kubernetes-validations": [
+            {
+                "message": "RequestRedirect filter must not be used together with backendRefs",
+                "rule": "(has(self.backendRefs) && size(self.backendRefs) > 0) ? (!has(self.filters) || self.filters.all(f, !has(f.requestRedirect))): true"
+            },
+            {
+                "message": "When using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.filters) && self.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == 'ReplacePrefixMatch' && has(f.requestRedirect.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            },
+            {
+                "message": "When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.filters) && self.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == 'ReplacePrefixMatch' && has(f.urlRewrite.path.replacePrefixMatch))) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            },
+            {
+                "message": "Within backendRefs, when using RequestRedirect filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.requestRedirect) && has(f.requestRedirect.path) && f.requestRedirect.path.type == 'ReplacePrefixMatch' && has(f.requestRedirect.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            },
+            {
+                "message": "Within backendRefs, When using URLRewrite filter with path.replacePrefixMatch, exactly one PathPrefix match must be specified",
+                "rule": "(has(self.backendRefs) && self.backendRefs.exists_one(b, (has(b.filters) && b.filters.exists_one(f, has(f.urlRewrite) && has(f.urlRewrite.path) && f.urlRewrite.path.type == 'ReplacePrefixMatch' && has(f.urlRewrite.path.replacePrefixMatch))) )) ? ((size(self.matches) != 1 || !has(self.matches[0].path) || self.matches[0].path.type != 'PathPrefix') ? false : true) : true"
+            }
+        ]
+    },
+    "maxItems": 16,
+    "type": "array"
+}
+
+GRPC_RULES_SCHEMA = {
+    "description": "Rules are a list of GRPC matchers, filters and actions.",
+    "items": {
+        "description": "GRPCRouteRule defines the semantics for matching a gRPC request based on\nconditions (matches), processing it (filters), and forwarding the request to\nan API object (backendRefs).",
+        "properties": {
+            "backendRefs": {
+                "description": "BackendRefs defines the backend(s) where matching requests should be\nsent.\n\n\nFailure behavior here depends on how many BackendRefs are specified and\nhow many are invalid.\n\n\nIf *all* entries in BackendRefs are invalid, and there are also no filters\nspecified in this route rule, *all* traffic which matches this rule MUST\nreceive an `UNAVAILABLE` status.\n\n\nSee the GRPCBackendRef definition for the rules about what makes a single\nGRPCBackendRef invalid.\n\n\nWhen a GRPCBackendRef is invalid, `UNAVAILABLE` statuses MUST be returned for\nrequests that would have otherwise been routed to an invalid backend. If\nmultiple backends are specified, and some are invalid, the proportion of\nrequests that would otherwise have been routed to an invalid backend\nMUST receive an `UNAVAILABLE` status.\n\n\nFor example, if two backends are specified with equal weights, and one is\ninvalid, 50 percent of traffic MUST receive an `UNAVAILABLE` status.\nImplementations may choose how that 50 percent is determined.\n\n\nSupport: Core for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource\n\n\nSupport for weight: Core",
+                "items": {
+                    "description": "GRPCBackendRef defines how a GRPCRoute forwards a gRPC request.\n\n\nNote that when a namespace different than the local namespace is specified, a\nReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\n<gateway:experimental:description>\n\n\nWhen the BackendRef points to a Kubernetes Service, implementations SHOULD\nhonor the appProtocol field if it is set for the target Service Port.\n\n\nImplementations supporting appProtocol SHOULD recognize the Kubernetes\nStandard Application Protocols defined in KEP-3726.\n\n\nIf a Service appProtocol isn't specified, an implementation MAY infer the\nbackend protocol through its own means. Implementations MAY infer the\nprotocol from the Route type referring to the backend Service.\n\n\nIf a Route is not able to send traffic to the backend using the specified\nprotocol then the backend is considered invalid. Implementations MUST set the\n\"ResolvedRefs\" condition to \"False\" with the \"UnsupportedProtocol\" reason.\n\n\n</gateway:experimental:description>",
+                    "properties": {
+                        "filters": {
+                            "description": "Filters defined at this level MUST be executed if and only if the\nrequest is being forwarded to the backend defined here.\n\n\nSupport: Implementation-specific (For broader support of filters, use the\nFilters field in GRPCRouteRule.)",
+                            "items": {
+                                "description": "GRPCRouteFilter defines processing steps that must be completed during the\nrequest or response lifecycle. GRPCRouteFilters are meant as an extension\npoint to express processing that may be done in Gateway implementations. Some\nexamples include request or response modification, implementing\nauthentication strategies, rate-limiting, and traffic shaping. API\nguarantee/conformance is defined based on the type of the filter.",
+                                "properties": {
+                                    "extensionRef": {
+                                        "description": "ExtensionRef is an optional, implementation-specific extension to the\n\"filter\" behavior.  For example, resource \"myroutefilter\" in group\n\"networking.example.net\"). ExtensionRef MUST NOT be used for core and\nextended filters.\n\n\nSupport: Implementation-specific\n\n\nThis filter can be used multiple times within the same rule.",
+                                        "properties": {
+                                            "group": {
+                                                "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                                "maxLength": 253,
+                                                "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                "type": "string"
+                                            },
+                                            "kind": {
+                                                "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                                "maxLength": 63,
+                                                "minLength": 1,
+                                                "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                "type": "string"
+                                            },
+                                            "name": {
+                                                "description": "Name is the name of the referent.",
+                                                "maxLength": 253,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "group",
+                                            "kind",
+                                            "name"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "requestHeaderModifier": {
+                                        "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "requestMirror": {
+                                        "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "backendRef": {
+                                                "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                                "properties": {
+                                                    "group": {
+                                                        "default": "",
+                                                        "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                                        "maxLength": 253,
+                                                        "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                                        "type": "string"
+                                                    },
+                                                    "kind": {
+                                                        "default": "Service",
+                                                        "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "name": {
+                                                        "description": "Name is the name of the referent.",
+                                                        "maxLength": 253,
+                                                        "minLength": 1,
+                                                        "type": "string"
+                                                    },
+                                                    "namespace": {
+                                                        "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                                        "maxLength": 63,
+                                                        "minLength": 1,
+                                                        "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                                        "type": "string"
+                                                    },
+                                                    "port": {
+                                                        "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                                        "format": "int32",
+                                                        "maximum": 65535,
+                                                        "minimum": 1,
+                                                        "type": "integer"
+                                                    }
+                                                },
+                                                "required": [
+                                                    "name"
+                                                ],
+                                                "type": "object",
+                                                "x-kubernetes-validations": [
+                                                    {
+                                                        "message": "Must have port for Service reference",
+                                                        "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                                    }
+                                                ]
+                                            }
+                                        },
+                                        "required": [
+                                            "backendRef"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "responseHeaderModifier": {
+                                        "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                                        "properties": {
+                                            "add": {
+                                                "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            },
+                                            "remove": {
+                                                "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                                "items": {
+                                                    "type": "string"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-type": "set"
+                                            },
+                                            "set": {
+                                                "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                                "items": {
+                                                    "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                                    "properties": {
+                                                        "name": {
+                                                            "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                            "maxLength": 256,
+                                                            "minLength": 1,
+                                                            "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                            "type": "string"
+                                                        },
+                                                        "value": {
+                                                            "description": "Value is the value of HTTP Header to be matched.",
+                                                            "maxLength": 4096,
+                                                            "minLength": 1,
+                                                            "type": "string"
+                                                        }
+                                                    },
+                                                    "required": [
+                                                        "name",
+                                                        "value"
+                                                    ],
+                                                    "type": "object"
+                                                },
+                                                "maxItems": 16,
+                                                "type": "array",
+                                                "x-kubernetes-list-map-keys": [
+                                                    "name"
+                                                ],
+                                                "x-kubernetes-list-type": "map"
+                                            }
+                                        },
+                                        "type": "object"
+                                    },
+                                    "type": {
+                                        "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations supporting GRPCRoute MUST support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` MUST be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\n",
+                                        "enum": [
+                                            "ResponseHeaderModifier",
+                                            "RequestHeaderModifier",
+                                            "RequestMirror",
+                                            "ExtensionRef"
+                                        ],
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "type"
+                                ],
+                                "type": "object",
+                                "x-kubernetes-validations": [
+                                    {
+                                        "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                                        "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                                        "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                                        "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                                        "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                                        "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                                        "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                                        "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                                    },
+                                    {
+                                        "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                                        "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                                    }
+                                ]
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "RequestHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                                },
+                                {
+                                    "message": "ResponseHeaderModifier filter cannot be repeated",
+                                    "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                                }
+                            ]
+                        },
+                        "group": {
+                            "default": "",
+                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                            "maxLength": 253,
+                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                            "type": "string"
+                        },
+                        "kind": {
+                            "default": "Service",
+                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                            "type": "string"
+                        },
+                        "name": {
+                            "description": "Name is the name of the referent.",
+                            "maxLength": 253,
+                            "minLength": 1,
+                            "type": "string"
+                        },
+                        "namespace": {
+                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                            "maxLength": 63,
+                            "minLength": 1,
+                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                            "type": "string"
+                        },
+                        "port": {
+                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                            "format": "int32",
+                            "maximum": 65535,
+                            "minimum": 1,
+                            "type": "integer"
+                        },
+                        "weight": {
+                            "default": 1,
+                            "description": "Weight specifies the proportion of requests forwarded to the referenced\nbackend. This is computed as weight/(sum of all weights in this\nBackendRefs list). For non-zero values, there may be some epsilon from\nthe exact proportion defined here depending on the precision an\nimplementation supports. Weight is not a percentage and the sum of\nweights does not need to equal 100.\n\n\nIf only one backend is specified and it has a weight greater than 0, 100%\nof the traffic is forwarded to that backend. If weight is set to 0, no\ntraffic should be forwarded for this entry. If unspecified, weight\ndefaults to 1.\n\n\nSupport for this field varies based on the context where used.",
+                            "format": "int32",
+                            "maximum": 1000000,
+                            "minimum": 0,
+                            "type": "integer"
+                        }
+                    },
+                    "required": [
+                        "name"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "Must have port for Service reference",
+                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "type": "array"
+            },
+            "filters": {
+                "description": "Filters define the filters that are applied to requests that match\nthis rule.\n\n\nThe effects of ordering of multiple behaviors are currently unspecified.\nThis can change in the future based on feedback during the alpha stage.\n\n\nConformance-levels at this level are defined based on the type of filter:\n\n\n- ALL core filters MUST be supported by all implementations that support\n  GRPCRoute.\n- Implementers are encouraged to support extended filters.\n- Implementation-specific custom filters have no API guarantees across\n  implementations.\n\n\nSpecifying the same filter multiple times is not supported unless explicitly\nindicated in the filter.\n\n\nIf an implementation can not support a combination of filters, it must clearly\ndocument that limitation. In cases where incompatible or unsupported\nfilters are specified and cause the `Accepted` condition to be set to status\n`False`, implementations may use the `IncompatibleFilters` reason to specify\nthis configuration error.\n\n\nSupport: Core",
+                "items": {
+                    "description": "GRPCRouteFilter defines processing steps that must be completed during the\nrequest or response lifecycle. GRPCRouteFilters are meant as an extension\npoint to express processing that may be done in Gateway implementations. Some\nexamples include request or response modification, implementing\nauthentication strategies, rate-limiting, and traffic shaping. API\nguarantee/conformance is defined based on the type of the filter.",
+                    "properties": {
+                        "extensionRef": {
+                            "description": "ExtensionRef is an optional, implementation-specific extension to the\n\"filter\" behavior.  For example, resource \"myroutefilter\" in group\n\"networking.example.net\"). ExtensionRef MUST NOT be used for core and\nextended filters.\n\n\nSupport: Implementation-specific\n\n\nThis filter can be used multiple times within the same rule.",
+                            "properties": {
+                                "group": {
+                                    "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                    "maxLength": 253,
+                                    "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                    "type": "string"
+                                },
+                                "kind": {
+                                    "description": "Kind is kind of the referent. For example \"HTTPRoute\" or \"Service\".",
+                                    "maxLength": 63,
+                                    "minLength": 1,
+                                    "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                    "type": "string"
+                                },
+                                "name": {
+                                    "description": "Name is the name of the referent.",
+                                    "maxLength": 253,
+                                    "minLength": 1,
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "group",
+                                "kind",
+                                "name"
+                            ],
+                            "type": "object"
+                        },
+                        "requestHeaderModifier": {
+                            "description": "RequestHeaderModifier defines a schema for a filter that modifies request\nheaders.\n\n\nSupport: Core",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "requestMirror": {
+                            "description": "RequestMirror defines a schema for a filter that mirrors requests.\nRequests are sent to the specified destination, but responses from\nthat destination are ignored.\n\n\nThis filter can be used multiple times within the same rule. Note that\nnot all implementations will be able to support mirroring to multiple\nbackends.\n\n\nSupport: Extended",
+                            "properties": {
+                                "backendRef": {
+                                    "description": "BackendRef references a resource where mirrored requests are sent.\n\n\nMirrored requests must be sent only to a single destination endpoint\nwithin this BackendRef, irrespective of how many endpoints are present\nwithin this BackendRef.\n\n\nIf the referent cannot be found, this BackendRef is invalid and must be\ndropped from the Gateway. The controller must ensure the \"ResolvedRefs\"\ncondition on the Route status is set to `status: False` and not configure\nthis backend in the underlying implementation.\n\n\nIf there is a cross-namespace reference to an *existing* object\nthat is not allowed by a ReferenceGrant, the controller must ensure the\n\"ResolvedRefs\"  condition on the Route is set to `status: False`,\nwith the \"RefNotPermitted\" reason and not configure this backend in the\nunderlying implementation.\n\n\nIn either error case, the Message of the `ResolvedRefs` Condition\nshould be used to provide more detail about the problem.\n\n\nSupport: Extended for Kubernetes Service\n\n\nSupport: Implementation-specific for any other resource",
+                                    "properties": {
+                                        "group": {
+                                            "default": "",
+                                            "description": "Group is the group of the referent. For example, \"gateway.networking.k8s.io\".\nWhen unspecified or empty string, core API group is inferred.",
+                                            "maxLength": 253,
+                                            "pattern": "^$|^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$",
+                                            "type": "string"
+                                        },
+                                        "kind": {
+                                            "default": "Service",
+                                            "description": "Kind is the Kubernetes resource kind of the referent. For example\n\"Service\".\n\n\nDefaults to \"Service\" when not specified.\n\n\nExternalName services can refer to CNAME DNS records that may live\noutside of the cluster and as such are difficult to reason about in\nterms of conformance. They also may not be safe to forward to (see\nCVE-2021-25740 for more information). Implementations SHOULD NOT\nsupport ExternalName Services.\n\n\nSupport: Core (Services with a type other than ExternalName)\n\n\nSupport: Implementation-specific (Services with type ExternalName)",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "name": {
+                                            "description": "Name is the name of the referent.",
+                                            "maxLength": 253,
+                                            "minLength": 1,
+                                            "type": "string"
+                                        },
+                                        "namespace": {
+                                            "description": "Namespace is the namespace of the backend. When unspecified, the local\nnamespace is inferred.\n\n\nNote that when a namespace different than the local namespace is specified,\na ReferenceGrant object is required in the referent namespace to allow that\nnamespace's owner to accept the reference. See the ReferenceGrant\ndocumentation for details.\n\n\nSupport: Core",
+                                            "maxLength": 63,
+                                            "minLength": 1,
+                                            "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$",
+                                            "type": "string"
+                                        },
+                                        "port": {
+                                            "description": "Port specifies the destination port number to use for this resource.\nPort is required when the referent is a Kubernetes Service. In this\ncase, the port number is the service port number, not the target port.\nFor other resources, destination port might be derived from the referent\nresource or this field.",
+                                            "format": "int32",
+                                            "maximum": 65535,
+                                            "minimum": 1,
+                                            "type": "integer"
+                                        }
+                                    },
+                                    "required": [
+                                        "name"
+                                    ],
+                                    "type": "object",
+                                    "x-kubernetes-validations": [
+                                        {
+                                            "message": "Must have port for Service reference",
+                                            "rule": "(size(self.group) == 0 && self.kind == 'Service') ? has(self.port) : true"
+                                        }
+                                    ]
+                                }
+                            },
+                            "required": [
+                                "backendRef"
+                            ],
+                            "type": "object"
+                        },
+                        "responseHeaderModifier": {
+                            "description": "ResponseHeaderModifier defines a schema for a filter that modifies response\nheaders.\n\n\nSupport: Extended",
+                            "properties": {
+                                "add": {
+                                    "description": "Add adds the given header(s) (name, value) to the request\nbefore the action. It appends to any existing values associated\nwith the header name.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  add:\n  - name: \"my-header\"\n    value: \"bar,baz\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: foo,bar,baz",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                },
+                                "remove": {
+                                    "description": "Remove the given header(s) from the HTTP request before the action. The\nvalue of Remove is a list of HTTP header names. Note that the header\nnames are case-insensitive (see\nhttps://datatracker.ietf.org/doc/html/rfc2616#section-4.2).\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header1: foo\n  my-header2: bar\n  my-header3: baz\n\n\nConfig:\n  remove: [\"my-header1\", \"my-header3\"]\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header2: bar",
+                                    "items": {
+                                        "type": "string"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-type": "set"
+                                },
+                                "set": {
+                                    "description": "Set overwrites the request with the given header (name, value)\nbefore the action.\n\n\nInput:\n  GET /foo HTTP/1.1\n  my-header: foo\n\n\nConfig:\n  set:\n  - name: \"my-header\"\n    value: \"bar\"\n\n\nOutput:\n  GET /foo HTTP/1.1\n  my-header: bar",
+                                    "items": {
+                                        "description": "HTTPHeader represents an HTTP Header name and value as defined by RFC 7230.",
+                                        "properties": {
+                                            "name": {
+                                                "description": "Name is the name of the HTTP Header to be matched. Name matching MUST be\ncase insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2).\n\n\nIf multiple entries specify equivalent header names, the first entry with\nan equivalent name MUST be considered for a match. Subsequent entries\nwith an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                                "maxLength": 256,
+                                                "minLength": 1,
+                                                "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                                "type": "string"
+                                            },
+                                            "value": {
+                                                "description": "Value is the value of HTTP Header to be matched.",
+                                                "maxLength": 4096,
+                                                "minLength": 1,
+                                                "type": "string"
+                                            }
+                                        },
+                                        "required": [
+                                            "name",
+                                            "value"
+                                        ],
+                                        "type": "object"
+                                    },
+                                    "maxItems": 16,
+                                    "type": "array",
+                                    "x-kubernetes-list-map-keys": [
+                                        "name"
+                                    ],
+                                    "x-kubernetes-list-type": "map"
+                                }
+                            },
+                            "type": "object"
+                        },
+                        "type": {
+                            "description": "Type identifies the type of filter to apply. As with other API fields,\ntypes are classified into three conformance levels:\n\n\n- Core: Filter types and their corresponding configuration defined by\n  \"Support: Core\" in this package, e.g. \"RequestHeaderModifier\". All\n  implementations supporting GRPCRoute MUST support core filters.\n\n\n- Extended: Filter types and their corresponding configuration defined by\n  \"Support: Extended\" in this package, e.g. \"RequestMirror\". Implementers\n  are encouraged to support extended filters.\n\n\n- Implementation-specific: Filters that are defined and supported by specific vendors.\n  In the future, filters showing convergence in behavior across multiple\n  implementations will be considered for inclusion in extended or core\n  conformance levels. Filter-specific configuration for such filters\n  is specified using the ExtensionRef field. `Type` MUST be set to\n  \"ExtensionRef\" for custom filters.\n\n\nImplementers are encouraged to define custom implementation types to\nextend the core API with implementation-specific behavior.\n\n\nIf a reference to a custom filter type cannot be resolved, the filter\nMUST NOT be skipped. Instead, requests that would have been processed by\nthat filter MUST receive a HTTP error response.\n\n\n",
+                            "enum": [
+                                "ResponseHeaderModifier",
+                                "RequestHeaderModifier",
+                                "RequestMirror",
+                                "ExtensionRef"
+                            ],
+                            "type": "string"
+                        }
+                    },
+                    "required": [
+                        "type"
+                    ],
+                    "type": "object",
+                    "x-kubernetes-validations": [
+                        {
+                            "message": "filter.requestHeaderModifier must be nil if the filter.type is not RequestHeaderModifier",
+                            "rule": "!(has(self.requestHeaderModifier) && self.type != 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestHeaderModifier must be specified for RequestHeaderModifier filter.type",
+                            "rule": "!(!has(self.requestHeaderModifier) && self.type == 'RequestHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be nil if the filter.type is not ResponseHeaderModifier",
+                            "rule": "!(has(self.responseHeaderModifier) && self.type != 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.responseHeaderModifier must be specified for ResponseHeaderModifier filter.type",
+                            "rule": "!(!has(self.responseHeaderModifier) && self.type == 'ResponseHeaderModifier')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be nil if the filter.type is not RequestMirror",
+                            "rule": "!(has(self.requestMirror) && self.type != 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.requestMirror must be specified for RequestMirror filter.type",
+                            "rule": "!(!has(self.requestMirror) && self.type == 'RequestMirror')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be nil if the filter.type is not ExtensionRef",
+                            "rule": "!(has(self.extensionRef) && self.type != 'ExtensionRef')"
+                        },
+                        {
+                            "message": "filter.extensionRef must be specified for ExtensionRef filter.type",
+                            "rule": "!(!has(self.extensionRef) && self.type == 'ExtensionRef')"
+                        }
+                    ]
+                },
+                "maxItems": 16,
+                "type": "array",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "RequestHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'RequestHeaderModifier').size() <= 1"
+                    },
+                    {
+                        "message": "ResponseHeaderModifier filter cannot be repeated",
+                        "rule": "self.filter(f, f.type == 'ResponseHeaderModifier').size() <= 1"
+                    }
+                ]
+            },
+            "matches": {
+                "description": "Matches define conditions used for matching the rule against incoming\ngRPC requests. Each match is independent, i.e. this rule will be matched\nif **any** one of the matches is satisfied.\n\n\nFor example, take the following matches configuration:\n\n\n```\nmatches:\n- method:\n    service: foo.bar\n  headers:\n    values:\n      version: 2\n- method:\n    service: foo.bar.v2\n```\n\n\nFor a request to match against this rule, it MUST satisfy\nEITHER of the two conditions:\n\n\n- service of foo.bar AND contains the header `version: 2`\n- service of foo.bar.v2\n\n\nSee the documentation for GRPCRouteMatch on how to specify multiple\nmatch conditions to be ANDed together.\n\n\nIf no matches are specified, the implementation MUST match every gRPC request.\n\n\nProxy or Load Balancer routing configuration generated from GRPCRoutes\nMUST prioritize rules based on the following criteria, continuing on\nties. Merging MUST not be done between GRPCRoutes and HTTPRoutes.\nPrecedence MUST be given to the rule with the largest number of:\n\n\n* Characters in a matching non-wildcard hostname.\n* Characters in a matching hostname.\n* Characters in a matching service.\n* Characters in a matching method.\n* Header matches.\n\n\nIf ties still exist across multiple Routes, matching precedence MUST be\ndetermined in order of the following criteria, continuing on ties:\n\n\n* The oldest Route based on creation timestamp.\n* The Route appearing first in alphabetical order by\n  \"{namespace}/{name}\".\n\n\nIf ties still exist within the Route that has been given precedence,\nmatching precedence MUST be granted to the first matching rule meeting\nthe above criteria.",
+                "items": {
+                    "description": "GRPCRouteMatch defines the predicate used to match requests to a given\naction. Multiple match types are ANDed together, i.e. the match will\nevaluate to true only if all conditions are satisfied.\n\n\nFor example, the match below will match a gRPC request only if its service\nis `foo` AND it contains the `version: v1` header:\n\n\n```\nmatches:\n  - method:\n    type: Exact\n    service: \"foo\"\n    headers:\n  - name: \"version\"\n    value \"v1\"\n\n\n```",
+                    "properties": {
+                        "headers": {
+                            "description": "Headers specifies gRPC request header matchers. Multiple match values are\nANDed together, meaning, a request MUST match all the specified headers\nto select the route.",
+                            "items": {
+                                "description": "GRPCHeaderMatch describes how to select a gRPC route by matching gRPC request\nheaders.",
+                                "properties": {
+                                    "name": {
+                                        "description": "Name is the name of the gRPC Header to be matched.\n\n\nIf multiple entries specify equivalent header names, only the first\nentry with an equivalent name MUST be considered for a match. Subsequent\nentries with an equivalent header name MUST be ignored. Due to the\ncase-insensitivity of header names, \"foo\" and \"Foo\" are considered\nequivalent.",
+                                        "maxLength": 256,
+                                        "minLength": 1,
+                                        "pattern": "^[A-Za-z0-9!#$%&'*+\\-.^_\\x60|~]+$",
+                                        "type": "string"
+                                    },
+                                    "type": {
+                                        "default": "Exact",
+                                        "description": "Type specifies how to match against the value of the header.",
+                                        "enum": [
+                                            "Exact",
+                                            "RegularExpression"
+                                        ],
+                                        "type": "string"
+                                    },
+                                    "value": {
+                                        "description": "Value is the value of the gRPC Header to be matched.",
+                                        "maxLength": 4096,
+                                        "minLength": 1,
+                                        "type": "string"
+                                    }
+                                },
+                                "required": [
+                                    "name",
+                                    "value"
+                                ],
+                                "type": "object"
+                            },
+                            "maxItems": 16,
+                            "type": "array",
+                            "x-kubernetes-list-map-keys": [
+                                "name"
+                            ],
+                            "x-kubernetes-list-type": "map"
+                        },
+                        "method": {
+                            "description": "Method specifies a gRPC request service/method matcher. If this field is\nnot specified, all services and methods will match.",
+                            "properties": {
+                                "method": {
+                                    "description": "Value of the method to match against. If left empty or omitted, will\nmatch all services.\n\n\nAt least one of Service and Method MUST be a non-empty string.",
+                                    "maxLength": 1024,
+                                    "type": "string"
+                                },
+                                "service": {
+                                    "description": "Value of the service to match against. If left empty or omitted, will\nmatch any service.\n\n\nAt least one of Service and Method MUST be a non-empty string.",
+                                    "maxLength": 1024,
+                                    "type": "string"
+                                },
+                                "type": {
+                                    "default": "Exact",
+                                    "description": "Type specifies how to match against the service and/or method.\nSupport: Core (Exact with service and method specified)\n\n\nSupport: Implementation-specific (Exact with method specified but no service specified)\n\n\nSupport: Implementation-specific (RegularExpression)",
+                                    "enum": [
+                                        "Exact",
+                                        "RegularExpression"
+                                    ],
+                                    "type": "string"
+                                }
+                            },
+                            "type": "object",
+                            "x-kubernetes-validations": [
+                                {
+                                    "message": "One or both of 'service' or 'method' must be specified",
+                                    "rule": "has(self.type) ? has(self.service) || has(self.method) : true"
+                                },
+                                {
+                                    "message": "service must only contain valid characters (matching ^(?i)\\.?[a-z_][a-z_0-9]*(\\.[a-z_][a-z_0-9]*)*$)",
+                                    "rule": "(!has(self.type) || self.type == 'Exact') && has(self.service) ? self.service.matches(r\"\"\"^(?i)\\.?[a-z_][a-z_0-9]*(\\.[a-z_][a-z_0-9]*)*$\"\"\"): true"
+                                },
+                                {
+                                    "message": "method must only contain valid characters (matching ^[A-Za-z_][A-Za-z_0-9]*$)",
+                                    "rule": "(!has(self.type) || self.type == 'Exact') && has(self.method) ? self.method.matches(r\"\"\"^[A-Za-z_][A-Za-z_0-9]*$\"\"\"): true"
+                                }
+                            ]
+                        }
+                    },
+                    "type": "object"
+                },
+                "maxItems": 8,
+                "type": "array"
+            },
+            "sessionPersistence": {
+                "description": "SessionPersistence defines and configures session persistence\nfor the route rule.\n\n\nSupport: Extended\n\n\n",
+                "properties": {
+                    "absoluteTimeout": {
+                        "description": "AbsoluteTimeout defines the absolute timeout of the persistent\nsession. Once the AbsoluteTimeout duration has elapsed, the\nsession becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "cookieConfig": {
+                        "description": "CookieConfig provides configuration settings that are specific\nto cookie-based session persistence.\n\n\nSupport: Core",
+                        "properties": {
+                            "lifetimeType": {
+                                "default": "Session",
+                                "description": "LifetimeType specifies whether the cookie has a permanent or\nsession-based lifetime. A permanent cookie persists until its\nspecified expiry time, defined by the Expires or Max-Age cookie\nattributes, while a session cookie is deleted when the current\nsession ends.\n\n\nWhen set to \"Permanent\", AbsoluteTimeout indicates the\ncookie's lifetime via the Expires or Max-Age cookie attributes\nand is required.\n\n\nWhen set to \"Session\", AbsoluteTimeout indicates the\nabsolute lifetime of the cookie tracked by the gateway and\nis optional.\n\n\nSupport: Core for \"Session\" type\n\n\nSupport: Extended for \"Permanent\" type",
+                                "enum": [
+                                    "Permanent",
+                                    "Session"
+                                ],
+                                "type": "string"
+                            }
+                        },
+                        "type": "object"
+                    },
+                    "idleTimeout": {
+                        "description": "IdleTimeout defines the idle timeout of the persistent session.\nOnce the session has been idle for more than the specified\nIdleTimeout duration, the session becomes invalid.\n\n\nSupport: Extended",
+                        "pattern": "^([0-9]{1,5}(h|m|s|ms)){1,4}$",
+                        "type": "string"
+                    },
+                    "sessionName": {
+                        "description": "SessionName defines the name of the persistent session token\nwhich may be reflected in the cookie or the header. Users\nshould avoid reusing session names to prevent unintended\nconsequences, such as rejection or unpredictable behavior.\n\n\nSupport: Implementation-specific",
+                        "maxLength": 128,
+                        "type": "string"
+                    },
+                    "type": {
+                        "default": "Cookie",
+                        "description": "Type defines the type of session persistence such as through\nthe use a header or cookie. Defaults to cookie based session\npersistence.\n\n\nSupport: Core for \"Cookie\" type\n\n\nSupport: Extended for \"Header\" type",
+                        "enum": [
+                            "Cookie",
+                            "Header"
+                        ],
+                        "type": "string"
+                    }
+                },
+                "type": "object",
+                "x-kubernetes-validations": [
+                    {
+                        "message": "AbsoluteTimeout must be specified when cookie lifetimeType is Permanent",
+                        "rule": "!has(self.cookieConfig.lifetimeType) || self.cookieConfig.lifetimeType != 'Permanent' || has(self.absoluteTimeout)"
+                    }
+                ]
+            }
+        },
+        "type": "object"
+    },
+    "maxItems": 16,
+    "type": "array",
+}
+
+SCHEMA = {
+  "anyOf": [
+    TCP_RULES_SCHEMA,
+    UDP_RULES_SCHEMA,
+    TLS_RULES_SCHEMA,
+    HTTP_RULES_SCHEMA,
+    GRPC_RULES_SCHEMA,
+  ],
+  "$schema": "http://json-schema.org/schema#"
+}
diff --git a/rootfs/api/serializers/schemas/volumes.py b/rootfs/api/serializers/schemas/volumes.py
new file mode 100644
index 000000000..1b44509de
--- /dev/null
+++ b/rootfs/api/serializers/schemas/volumes.py
@@ -0,0 +1,24 @@
+SCHEMA = {
+    "$schema": "http://json-schema.org/schema#",
+    "type": "object",
+    "properties": {
+        "nfs": {
+            "type": "object",
+            "properties": {
+                "server": {"type": "string"},
+                "path": {"type": "string"},
+            },
+            "required": ["server", "path"],
+        },
+        "oss": {
+            "type": "object",
+            "properties": {
+                "bucket": {"type": "string"},
+                "server": {"type": "string"},
+                "access_key": {"type": "string"},
+                "secret_key": {"type": "string"},
+            },
+            "required": ["bucket", "server", "access_key", "secret_key"],
+        },
+    },
+}
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
index bdf000ddb..4a9183d94 100644
--- a/rootfs/api/settings/celery.py
+++ b/rootfs/api/settings/celery.py
@@ -5,7 +5,6 @@
 
 class Config(object):
     # Celery Configuration Options
-    timezone = "Asia/Shanghai"
     enable_utc = True
     task_serializer = 'pickle'
     accept_content = frozenset([
@@ -17,12 +16,15 @@ class Config(object):
     task_track_started = True
     task_time_limit = 30 * 60
     worker_max_tasks_per_child = 200
+    worker_prefetch_multiplier = 1
     result_expires = 24 * 60 * 60
-    broker_url = os.environ.get('DRYCC_RABBITMQ_URL', 'amqp://guest:guest@127.0.0.1:5672/')  # noqa
     cache_backend = 'django-cache'
-    task_default_queue = 'low'
+    task_default_queue = 'controller.middle'
     task_default_exchange = 'controller.priority'
-    task_default_routing_key = 'controller.priority.low'
+    task_default_routing_key = 'controller.priority.middle'
+    broker_transport_options = {"queue_order_strategy": "sorted"}
+    task_create_missing_queues = True
+    task_inherit_parent_priority = True
     broker_connection_retry_on_startup = True
     worker_cancel_long_running_tasks_on_connection_loss = True
 
@@ -30,47 +32,52 @@ class Config(object):
 app = Celery('drycc')
 app.config_from_object(Config())
 app.conf.update(
+    timezone=os.environ.get('TZ', 'UTC'),
     task_routes={
+        'api.tasks.send_usage': {
+            'queue': 'controller.low',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.low',
+        },
         'api.tasks.scale_app': {
-            'queue': 'high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'queue': 'controller.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
+        },
+        'api.tasks.mount_app': {
+            'queue': 'controller.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
         'api.tasks.restart_app': {
-            'queue': 'high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+            'queue': 'controller.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
-        'api.tasks.retrieve_resource': {
-            'queue': 'high',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.high',
+        'api.tasks.run_pipeline': {
+            'queue': 'controller.high',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.high',
         },
-        'api.tasks.send_measurements': {
-            'queue': 'middle',
-            'exchange': 'controller.priority',
-            'routing_key': 'controller.priority.middle',
+        'api.tasks.send_app_log': {
+            'queue': 'controller.middle',
+            'exchange': 'controller.priority', 'routing_key': 'controller.priority.middle',
         },
     },
     task_queues=(
         Queue(
-            'low',
-            exchange=Exchange('controller.priority', type="direct"),
+            'controller.low', exchange=Exchange('controller.priority', type="direct"),
             routing_key='controller.priority.low',
-            queue_arguments={'x-max-priority': 16},
         ),
         Queue(
-            'high',
-            exchange=Exchange('controller.priority', type="direct"),
+            'controller.high', exchange=Exchange('controller.priority', type="direct"),
             routing_key='controller.priority.high',
-            queue_arguments={'x-max-priority': 64},
         ),
         Queue(
-            'middle',
-            exchange=Exchange('controller.priority', type="direct"),
+            'controller.middle', exchange=Exchange('controller.priority', type="direct"),
             routing_key='controller.priority.middle',
-            queue_arguments={'x-max-priority': 32},
         ),
     ),
 )
+DRYCC_VALKEY_URL = os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379')
+app.conf.update(
+    broker_url=DRYCC_VALKEY_URL,
+    result_backend=DRYCC_VALKEY_URL,
+    broker_transport_options={"queue_order_strategy": "sorted", "visibility_timeout": 43200},
+)
 app.autodiscover_tasks()
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
index 7cb94ccbc..7f2d43b88 100644
--- a/rootfs/api/settings/production.py
+++ b/rootfs/api/settings/production.py
@@ -1,19 +1,28 @@
 """
 Django settings for the Drycc project.
 """
-from distutils.util import strtobool
 import sys
-import os.path
-import tempfile
+import uuid
 import json
+import random
+import string
+import os.path
 import dj_database_url
 
 
+def randstr(k):
+    return ''.join(random.choices(string.ascii_lowercase + string.digits, k=k))
+
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.insert(0, os.path.join(BASE_DIR, 'apps_extra'))
+
+# drycc controller app version.
+VERSION = os.environ.get('VERSION', uuid.uuid1().hex[:8])
+
 # A boolean that turns on/off debug mode.
 # https://docs.djangoproject.com/en/1.11/ref/settings/#debug
-DEBUG = bool(strtobool(os.environ.get('DRYCC_DEBUG', 'false')))
+DEBUG = os.environ.get('DRYCC_DEBUG', 'false').lower() == "true"
 
 # If set to True, Django's normal exception handling of view functions
 # will be suppressed, and exceptions will propagate upwards
@@ -63,6 +72,7 @@
         'BACKEND': 'django.template.backends.django.DjangoTemplates',
         'DIRS': [
             # insert your TEMPLATE_DIRS here
+            os.path.join(BASE_DIR, "templates"),
         ],
         'APP_DIRS': True,
         'OPTIONS': {
@@ -88,7 +98,7 @@
     'django.middleware.common.CommonMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
-    'api.middleware.APIVersionMiddleware',
+    'api.middleware.DjangoAPIVersionMiddleware',
 ]
 
 ROOT_URLCONF = 'drycc.urls'
@@ -104,10 +114,8 @@
     'django.contrib.sessions',
     # Third-party apps
     'corsheaders',
-    'guardian',
     'gunicorn',
     'rest_framework',
-    'rest_framework.authtoken',
     'social_django',
     # Drycc apps
     'api'
@@ -118,10 +126,8 @@
 
 AUTHENTICATION_BACKENDS = (
     "django.contrib.auth.backends.ModelBackend",
-    "guardian.backends.ObjectPermissionBackend",
+    "api.apps_extra.social_core.backends.DryccOIDC",
 )
-GUARDIAN_GET_INIT_ANONYMOUS_USER = 'api.models.base.get_anonymous_user_instance'
-ANONYMOUS_USER_NAME = os.environ.get('ANONYMOUS_USER_NAME', 'AnonymousUser')
 LOGIN_URL = '/v2/auth/login/'
 
 # Security settings
@@ -144,8 +150,8 @@
 CSRF_COOKIE_SAMESITE = None
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_BROWSER_XSS_FILTER = True
-SESSION_COOKIE_SECURE = bool(strtobool(os.environ.get('SESSION_COOKIE_SECURE', 'false')))
-CSRF_COOKIE_SECURE = bool(strtobool(os.environ.get('CSRF_COOKIE_SECURE', 'false')))
+SESSION_COOKIE_SECURE = os.environ.get('SESSION_COOKIE_SECURE', 'false').lower() == "true"
+CSRF_COOKIE_SECURE = os.environ.get('CSRF_COOKIE_SECURE', 'false').lower() == "true"
 
 
 # Honor HTTPS from a trusted proxy
@@ -170,7 +176,15 @@
     'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination',
     'PAGE_SIZE': 100,
     'TEST_REQUEST_DEFAULT_FORMAT': 'json',
-    'EXCEPTION_HANDLER': 'api.exceptions.custom_exception_handler'
+    'EXCEPTION_HANDLER': 'api.exceptions.custom_exception_handler',
+    'DEFAULT_THROTTLE_CLASSES': [
+        'rest_framework.throttling.AnonRateThrottle',
+        'rest_framework.throttling.UserRateThrottle'
+    ],
+    'DEFAULT_THROTTLE_RATES': {
+        'anon': os.environ.get('DRYCC_CONTROLLER_THROTTLE_RATES_ANON', '30/min'),
+        'user': os.environ.get('DRYCC_CONTROLLER_THROTTLE_RATES_USER', '60/min'),
+    }
 }
 
 # URLs that end with slashes are ugly
@@ -229,83 +243,111 @@
         },
         'api': {
             'handlers': ['console'],
-            'propagate': True,
-        },
-        'registry': {
-            'handlers': ['console'],
+            'level': 'INFO',
             'propagate': True,
         },
         'scheduler': {
             'handlers': ['console'],
+            'level': 'INFO',
             'propagate': True,
         },
     }
 }
 TEST_RUNNER = 'api.tests.SilentDjangoTestSuiteRunner'
 
-# default drycc settings
-LOG_LINES = 100
-TEMPDIR = tempfile.mkdtemp(prefix='drycc')
+# drycc controller runner(api, metric, mutate)
+RUNNER = os.environ.get('DRYCC_CONTROLLER_RUNNER', 'api')
 
 # names which apps cannot reserve for routing
-DRYCC_RESERVED_NAMES = os.environ.get('RESERVED_NAMES', '').replace(' ', '').split(',')
-DRYCC_RESERVED_NAMES.append("canary")
+RESERVED_NAME_PATTERNS_PATH = os.environ.get(
+    'RESERVED_NAME_PATTERNS_PATH', '/etc/controller/reserved-name-patterns.txt')
+if os.path.exists(RESERVED_NAME_PATTERNS_PATH):
+    with open(RESERVED_NAME_PATTERNS_PATH) as f:
+        RESERVED_NAME_PATTERNS = [line.strip() for line in f if line.strip()]
+else:
+    RESERVED_NAME_PATTERNS = [r"^drycc(?:-[\w-]+)?$", r"^kube(?:-[\w-]+)?$", r"^default$"]
 
 # the k8s namespace in which the controller and workflow were installed.
 WORKFLOW_NAMESPACE = os.environ.get('WORKFLOW_NAMESPACE', 'drycc')
 
-# kubernetes cluster domain
-KUBERNETES_CLUSTER_DOMAIN = os.environ.get("KUBERNETES_CLUSTER_DOMAIN", "cluster.local")
 # default scheduler settings
 SCHEDULER_MODULE = 'scheduler'
 SCHEDULER_URL = "https://{}:{}".format(
-    # accessing the k8s api server by IP address rather than hostname avoids
-    # intermittent DNS errors
-    os.environ.get(
-        'KUBERNETES_SERVICE_HOST',
-        'kubernetes.default.svc.{}'.format(KUBERNETES_CLUSTER_DOMAIN)
-    ),
-    os.environ.get('KUBERNETES_SERVICE_PORT', '443')
+    os.environ.get('KUBERNETES_SERVICE_HOST', 'kubernetes.default'),
+    os.environ.get('KUBERNETES_SERVICE_PORT', '443'),
 )
 
-K8S_API_VERIFY_TLS = bool(strtobool(os.environ.get('K8S_API_VERIFY_TLS', 'true')))
-# drycc prometheus url
-DRYCC_PROMETHEUS_URL = os.environ.get('DRYCC_PROMETHEUS_URL', '')
-
-# security keys and auth tokens
-random_secret = 'CHANGEME_sapm$s%upvsw5l_zuy_&29rkywd^78ff(qi*#@&*^'
-SECRET_KEY = os.environ.get('DRYCC_SECRET_KEY', random_secret)
-BUILDER_KEY = os.environ.get('DRYCC_BUILDER_KEY', random_secret)
-
-# gateway class name
-GATEWAY_CLASS = os.environ.get('DRYCC_GATEWAY_CLASS', '')
+K8S_API_VERIFY_TLS = os.environ.get('K8S_API_VERIFY_TLS', 'true').lower() == "true"
+
+# drycc victoriametrics url
+DRYCC_VICTORIAMETRICS_URL = os.environ.get('DRYCC_VICTORIAMETRICS_URL', '')
+
+# drycc metrics config file
+DRYCC_METRICS_CONFIG = {}
+DRYCC_METRICS_CONFIG_PATH = os.environ.get(
+    'DRYCC_METRICS_CONFIG_PATH', '/etc/controller/metrics.json')
+if os.path.exists(DRYCC_METRICS_CONFIG_PATH):
+    with open(DRYCC_METRICS_CONFIG_PATH) as fd:
+        DRYCC_METRICS_CONFIG = json.load(fd)
+DRYCC_METRICS_EXPIRY = int(os.environ.get('DRYCC_METRICS_EXPIRY', '20'))
+DRYCC_METRICS_INTERVAL = os.environ.get('DRYCC_METRICS_INTERVAL', '5m')
+
+# drycc volume template
+DRYCC_VOLUME_TEMPLATE = {}
+DRYCC_VOLUME_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_VOLUME_TEMPLATE_PATH', '/etc/controller/volume-template.json')
+if os.path.exists(DRYCC_VOLUME_TEMPLATE_PATH):
+    with open(DRYCC_VOLUME_TEMPLATE_PATH) as fd:
+        DRYCC_VOLUME_TEMPLATE = fd.read()
+
+# drycc volume claim template
+DRYCC_VOLUME_CLAIM_TEMPLATE = {}
+DRYCC_VOLUME_CLAIM_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_VOLUME_CLAIM_TEMPLATE_PATH', '/etc/controller/volume-claim-template.json')
+if os.path.exists(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH):
+    with open(DRYCC_VOLUME_CLAIM_TEMPLATE_PATH) as fd:
+        DRYCC_VOLUME_CLAIM_TEMPLATE = fd.read()
+
+# drycc volume usage template
+DRYCC_VOLUME_USAGE_TEMPLATE = ""
+DRYCC_VOLUME_USAGE_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_VOLUME_USAGE_TEMPLATE_PATH', '/etc/controller/volume-usage-template.promql')
+if os.path.exists(DRYCC_VOLUME_USAGE_TEMPLATE_PATH):
+    with open(DRYCC_VOLUME_USAGE_TEMPLATE_PATH) as fd:
+        DRYCC_VOLUME_USAGE_TEMPLATE = fd.read()
+
+# drycc network usage template
+DRYCC_NETWORK_USAGE_TEMPLATE = ""
+DRYCC_NETWORK_USAGE_TEMPLATE_PATH = os.environ.get(
+    'DRYCC_NETWORK_USAGE_TEMPLATE_PATH', '/etc/controller/network-usage-template.promql')
+if os.path.exists(DRYCC_NETWORK_USAGE_TEMPLATE_PATH):
+    with open(DRYCC_NETWORK_USAGE_TEMPLATE_PATH) as fd:
+        DRYCC_NETWORK_USAGE_TEMPLATE = fd.read()
+
+# Django secret key
+SECRET_KEY = os.environ.get('DRYCC_SECRET_KEY', randstr(64))
+
+# Drycc service key
+SERVICE_KEY = os.environ.get('DRYCC_SERVICE_KEY', randstr(64))
+
+# Drycc cert key
+CERT_KEY_PATH = os.environ.get('DRYCC_CERT_KEY_PATH', '/etc/controller/cert/key')
+if os.path.exists(CERT_KEY_PATH):
+    with open(CERT_KEY_PATH) as f:
+        CERT_KEY = f.read()
+else:
+    CERT_KEY = None
 
 IMAGE_PULL_POLICY = os.environ.get('IMAGE_PULL_POLICY', "IfNotPresent")
 
-# True, true, yes, y and more evaluate to True
-# False, false, no, n and more evaluate to False
-# https://docs.python.org/3/distutils/apiref.html?highlight=distutils.util#distutils.util.strtobool
-# see the above for all available options
-#
-# If a user deploys one build with a Procfile but then forgets to in the next one
-# then let that go through without scaling the missing process types down
-#
-# If the user has a Procfile in both deploys then processes are scaled up / down as per usual
-#
-# By default the process types are scaled down unless this setting is turned on
-DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE = bool(strtobool(os.environ.get(
-    'DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE', 'true')))
-
-# True, true, yes, y and more evaluate to True
-# False, false, no, n and more evaluate to False
-# https://docs.python.org/3/distutils/apiref.html?highlight=distutils.util#distutils.util.strtobool
-# see the above for all available options
-#
-# If a previous deploy had a Procfile but then the following deploy has no Procfile then it will
-# result in a 406 - Not Acceptable
-# Has priority over DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE
-DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING = bool(strtobool(os.environ.get(
-    'DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING', 'false')))
+# apply task size
+DRYCC_APPLY_TASKS = int(os.environ.get('DRYCC_APPLY_TASKS', '20'))
+
+# Drycc filer image
+# Provide get and put operations for `drycc volumes:client`
+DRYCC_FILER_IMAGE = os.environ.get('DRYCC_FILER_IMAGE', 'registry.drycc.cc/drycc/filer:canary')
+DRYCC_FILER_IMAGE_PULL_POLICY = os.environ.get('DRYCC_FILER_IMAGE_PULL_POLICY', 'IfNotPresent')
+DRYCC_FILER_DURATION = int(os.environ.get('DRYCC_FILER_DURATION', '3600'))
 
 # Define a global default on how many pods to bring up and then
 # take down sequentially during a deploy
@@ -320,6 +362,9 @@
 # where it roughly goes BATCHES * TIMEOUT = global timeout
 DRYCC_DEPLOY_TIMEOUT = int(os.environ.get('DRYCC_DEPLOY_TIMEOUT', 120))
 
+# the timeout of the pipeline run phase
+DRYCC_PILELINE_RUN_TIMEOUT = int(os.environ.get('DRYCC_PILELINE_RUN_TIMEOUT', 3600))
+
 try:
     DRYCC_DEPLOY_HOOK_URLS = os.environ['DRYCC_DEPLOY_HOOK_URLS'].split(',')
 except KeyError:
@@ -327,96 +372,33 @@
 
 DRYCC_DEPLOY_HOOK_SECRET_KEY = os.environ.get('DRYCC_DEPLOY_HOOK_SECRET_KEY', None)
 
-DRYCC_APP_STORAGE_CLASS = os.environ.get('DRYCC_APP_STORAGE_CLASS', "")
+DRYCC_APP_GATEWAY_CLASS = os.environ.get('DRYCC_APP_GATEWAY_CLASS', "")
 
-DRYCC_APP_RUNTIME_CLASS = os.environ.get('DRYCC_APP_RUNTIME_CLASS', "")
+DRYCC_APP_STORAGE_CLASS = os.environ.get('DRYCC_APP_STORAGE_CLASS', "")
 
 DRYCC_APP_DNS_POLICY = os.environ.get('DRYCC_APP_DNS_POLICY', "")
 
 DRYCC_APP_POD_EXEC_TIMEOUT = int(os.environ.get('DRYCC_APP_POD_EXEC_TIMEOUT', "3600"))
 
-DRYCC_DEFAULT_CONFIG_TAGS = os.environ.get('DRYCC_DEFAULT_CONFIG_TAGS', '')
-
 KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT = os.environ.get(
     'KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', None)
 
 # How long k8s waits for a pod to finish work after a SIGTERM before sending SIGKILL
 KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(os.environ.get(
     'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 30))
-
-# min and max memory for a CPU
-KUBERNETES_CPU_MEMORY_RATIO = tuple(map(lambda x: x.strip(), os.environ.get(
-    'DRYCC_CPU_MEMORY_DISTRIBUTION_RATIO', '1G,4G').split(",")))
-# CPU request ratio
-KUBERNETES_REQUEST_CPU_RATIO = int(os.environ.get('KUBERNETES_REQUEST_CPU_RATIO', '2'))
-# Memory request ratio
-KUBERNETES_REQUEST_MEMORY_RATIO = int(os.environ.get('KUBERNETES_REQUEST_MEMORY_RATIO', '2'))
-
-# Minimum CPU limit, units are represented in the millicpu of CPUs
-KUBERNETES_LIMITS_MIN_CPU = int(os.environ.get('KUBERNETES_LIMITS_MIN_CPU', '125'))
-# Max CPU limit, units are represented in the millicpu of CPUs
-KUBERNETES_LIMITS_MAX_CPU = int(os.environ.get('KUBERNETES_LIMITS_MAX_CPU', 32 * 1000))
-# Minimum Memory limit, units are represented in Megabytes(M)
-KUBERNETES_LIMITS_MIN_MEMORY = int(os.environ.get('KUBERNETES_LIMITS_MIN_MEMORY', '128'))
-# Max Memory limit, units are represented in Megabytes(M)
-KUBERNETES_LIMITS_MAX_MEMORY = int(os.environ.get('KUBERNETES_LIMITS_MAX_MEMORY', 128 * 1024))
 # Minimum Stroage Volume limit, units are represented in Gigabytes(G)
 KUBERNETES_LIMITS_MIN_VOLUME = int(os.environ.get('KUBERNETES_LIMITS_MIN_VOLUME', 1))
 # Max Stroage Volume limit, units are represented in Gigabytes(G)
 KUBERNETES_LIMITS_MAX_VOLUME = int(os.environ.get('KUBERNETES_LIMITS_MAX_VOLUME', 1024 * 16))
-# Default pod spec volumes for application.
-KUBERNETES_POD_DEFAULT_VOLUMES = os.environ.get('KUBERNETES_POD_DEFAULT_VOLUMES', '[]')
-# Default pod spec volume mounts for application.
-KUBERNETES_POD_DEFAULT_VOLUME_MOUNTS = os.environ.get('KUBERNETES_POD_DEFAULT_VOLUME_MOUNTS', '[]')
-# Default pod spec security context for application.
-KUBERNETES_POD_DEFAULT_SECURITY_CONTEXT = os.environ.get(
-    'KUBERNETES_POD_DEFAULT_SECURITY_CONTEXT',
-    json.dumps({
-        'capabilities': {
-            'add': ['SYS_ADMIN']
-        },
-        'allowPrivilegeEscalation': True,
-    })
-)
 
-# Default pod spec resources for application.
-KUBERNETES_POD_DEFAULT_RESOURCES = os.environ.get(
-    'KUBERNETES_POD_DEFAULT_RESOURCES',
-    json.dumps({
-        "requests": {
-            "ephemeral-storage": "1Gi",
-        },
-        "limits": {
-            "ephemeral-storage": "2Gi",
-        }
-    })
-)
-
-# Default pod annotations for application.
-KUBERNETES_POD_DEFAULT_ANNOTATIONS = os.environ.get(
-    'KUBERNETES_POD_DEFAULT_ANNOTATIONS',
-    json.dumps({
-        "kubernetes.io/egress-bandwidth": "100M",
-        "kubernetes.io/ingress-bandwidth": "100M",
-    })
-)
-# Default quota spec for application namespace
-KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC = os.environ.get(
-    'KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC', ''
-)
-# Default limit range spec for application namespace
-KUBERNETES_NAMESPACE_DEFAULT_LIMIT_RANGES_SPEC = os.environ.get(
-    'KUBERNETES_NAMESPACE_DEFAULT_LIMIT_RANGES_SPEC', ''
-)
+KUBERNETES_JOB_MAX_TTL_SECONDS_AFTER_FINISHED = int(os.environ.get(
+    'KUBERNETES_JOB_MAX_TTL_SECONDS_AFTER_FINISHED', '7200'
+))
 
 # registry settings
 REGISTRY_LOCATION = os.environ.get('DRYCC_REGISTRY_LOCATION', 'on-cluster')
 REGISTRY_SECRET_PREFIX = os.environ.get('DRYCC_REGISTRY_SECRET_PREFIX', 'private-registry')
 
-# logger settings
-LOGGER_HOST = os.environ.get('DRYCC_LOGGER_SERVICE_HOST', '127.0.0.1')
-LOGGER_PORT = os.environ.get('DRYCC_LOGGER_SERVICE_PORT_HTTP', 80)
-
 DRYCC_DATABASE_URL = os.environ.get('DRYCC_DATABASE_URL', 'postgres://postgres:@:5432/drycc')
 DATABASES = {
     'default': dj_database_url.config(default=DRYCC_DATABASE_URL)
@@ -427,60 +409,43 @@
 if DRYCC_DATABASE_REPLICA_URL is not None:
     DATABASES["replica"] = dj_database_url.config(default=DRYCC_DATABASE_REPLICA_URL)
 
-# database monitor setting
-DRYCC_DATABASE_MONITOR_URL = os.environ.get('DRYCC_DATABASE_MONITOR_URL', None)
-if DRYCC_DATABASE_MONITOR_URL is not None:
-    DATABASES["monitor"] = dj_database_url.config(default=DRYCC_DATABASE_MONITOR_URL)
-
 # database routers
-DATABASE_ROUTERS = ['api.routers.DefaultReplicaRouter', ]
+# Implements: 'api.routers.DefaultReplicaRouter'
+DATABASE_ROUTERS = [r for r in os.environ.get('DRYCC_DATABASE_ROUTERS', '').split(',') if r]
+
 
 APP_URL_REGEX = '[a-z0-9-]+'
 
+DOMAIN_URL_REGEX = r'\**\.?[-\._\w]+'
+
+NAME_REGEX = r'[a-z0-9]+(\-[a-z0-9]+)*'
+
 # Oauth settings
 
 DRYCC_PASSPORT_URL = os.environ.get('DRYCC_PASSPORT_URL', 'https://127.0.0.1:8000')
+DRYCC_REGISTER_URL = os.environ.get(
+    'DRYCC_REGISTER_URL',
+    f'{DRYCC_PASSPORT_URL}/user/registration'
+)
 
 LOGIN_REDIRECT_URL = os.environ.get(
     'LOGIN_REDIRECT_URL',
     f'{DRYCC_PASSPORT_URL}/user/login/done/',
 )
 
+# social auth settings
 SOCIAL_AUTH_DRYCC_KEY = os.environ.get(
     "DRYCC_PASSPORT_KEY",
     os.environ.get("SOCIAL_AUTH_DRYCC_KEY"),
 )
-
 SOCIAL_AUTH_DRYCC_SECRET = os.environ.get(
     'DRYCC_PASSPORT_SECRET',
     os.environ.get("SOCIAL_AUTH_DRYCC_SECRET"),
 )
-
-SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_AUTHORIZATION_URL',
-    f'{DRYCC_PASSPORT_URL}/oauth/authorize/',
-)
-SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_ACCESS_TOKEN_URL',
-    f'{DRYCC_PASSPORT_URL}/oauth/token/'
-)
-SOCIAL_AUTH_DRYCC_ACCESS_API_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_ACCESS_API_URL',
-    f'{DRYCC_PASSPORT_URL}'
-)
-SOCIAL_AUTH_DRYCC_USERINFO_URL = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_USERINFO_URL',
-    f'{DRYCC_PASSPORT_URL}/oauth/userinfo/'
-)
-SOCIAL_AUTH_DRYCC_JWKS_URI = os.environ.get(
-    'SOCIAL_AUTH_DRYCC_JWKS_URI',
-    f'{DRYCC_PASSPORT_URL}/oauth/.well-known/jwks.json'
-)
 SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT = os.environ.get(
     'SOCIAL_AUTH_DRYCC_OIDC_ENDPOINT',
     f'{DRYCC_PASSPORT_URL}/oauth'
 )
-
 SOCIAL_AUTH_JSONFIELD_ENABLED = True
 SOCIAL_AUTH_PIPELINE = (
     'social_core.pipeline.social_auth.social_details',
@@ -488,40 +453,31 @@
     'social_core.pipeline.social_auth.social_user',
     'social_core.pipeline.user.get_username',
     'social_core.pipeline.social_auth.associate_by_email',
-    'api.pipeline.update_or_create',
+    'api.apps_extra.social_core.pipelines.update_or_create',
     'social_core.pipeline.social_auth.associate_user',
     'social_core.pipeline.social_auth.load_extra_data',
     'social_core.pipeline.user.user_details',
 )
-AUTHENTICATION_BACKENDS = ("api.backend.DryccOIDC", ) + AUTHENTICATION_BACKENDS
-OAUTH_CACHE_USER_TIME = int(os.environ.get('OAUTH_CACHE_USER_TIME', 30 * 60))
+DRYCC_CACHE_USER_TIME = int(os.environ.get('DRYCC_CACHE_USER_TIME', 30 * 60))
 
-# Redis Configuration
-DRYCC_REDIS_ADDRS = os.environ.get('DRYCC_REDIS_ADDRS', '127.0.0.1:6379').split(",")
-DRYCC_REDIS_PASSWORD = os.environ.get('DRYCC_REDIS_PASSWORD', '')
+# Rate limit for invitation emails: max LIMIT emails per WINDOW seconds per recipient address
+DRYCC_INVITATION_EMAIL_LIMIT = int(os.environ.get('DRYCC_INVITATION_EMAIL_LIMIT', 5))
+DRYCC_INVITATION_EMAIL_TIMEOUT = int(os.environ.get('DRYCC_INVITATION_EMAIL_TIMEOUT', 3600))
 
-# Cache Configuration
+# Cache Valkey Configuration
 CACHES = {
     "default": {
-        "BACKEND": "django_redis.cache.RedisCache",
-        "LOCATION": [
-            'redis://:{}@{}'.format(
-                DRYCC_REDIS_PASSWORD, DRYCC_REDIS_ADDR) for DRYCC_REDIS_ADDR in DRYCC_REDIS_ADDRS
-        ],
-        "OPTIONS": {
-            "CLIENT_CLASS": "django_redis.client.ShardClient",
-        }
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "LOCATION": os.environ.get('DRYCC_VALKEY_URL', 'redis://:@127.0.0.1:6379'),
     }
 }
 
+# Quickwit Configuration
+QUICKWIT_INDEXER_URL = os.environ.get('QUICKWIT_INDEXER_URL', None)
+QUICKWIT_SEARCHER_URL = os.environ.get('QUICKWIT_SEARCHER_URL', None)
+QUICKWIT_LOG_INDEX_PREFIX = os.environ.get('QUICKWIT_LOG_INDEX_PREFIX', None)
+
 # Workflow-manager Configuration Options
 WORKFLOW_MANAGER_URL = os.environ.get('WORKFLOW_MANAGER_URL', None)
 WORKFLOW_MANAGER_ACCESS_KEY = os.environ.get('WORKFLOW_MANAGER_ACCESS_KEY', None)
 WORKFLOW_MANAGER_SECRET_KEY = os.environ.get('WORKFLOW_MANAGER_SECRET_KEY', None)
-
-# Drycc admission webhook token
-if os.path.exists("/etc/controller/webhook/cert"):
-    with open("/etc/controller/webhook/cert/token") as f:
-        DRYCC_ADMISSION_WEBHOOK_TOKEN = f.read()
-else:
-    DRYCC_ADMISSION_WEBHOOK_TOKEN = None
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
index 6b2b37d04..25abe6502 100644
--- a/rootfs/api/settings/testing.py
+++ b/rootfs/api/settings/testing.py
@@ -1,11 +1,20 @@
 import random
 import string
 import os
+import multiprocessing
 from django.core import signals
 from api.settings.celery import app
-from api.settings.production import DATABASES
+from api.settings.production import DATABASES, REST_FRAMEWORK
 from api.settings.production import *  # noqa
 
+
+# Disable DRF throttling for unit tests
+del REST_FRAMEWORK['DEFAULT_THROTTLE_RATES']
+del REST_FRAMEWORK['DEFAULT_THROTTLE_CLASSES']
+# Fix Django test error.
+# https://github.com/django/django/blob/main/django/test/runner.py#L455
+# This code was removed in Django > 5.2, so it is safe to remove this when we upgrade.
+multiprocessing.set_start_method('fork')
 # Monkey patch celery
 app.conf.update(task_always_eager=True)
 signals.request_started.send = lambda sender, **named: []
@@ -40,10 +49,8 @@
 }
 
 # How long k8s waits for a pod to finish work after a SIGTERM before sending SIGKILL
-KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(os.environ.get('KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 2))  # noqa
-KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC = '{"spec":{"hard":{"pods":"10"}}}'
-
-DRYCC_DEFAULT_CONFIG_TAGS = os.environ.get('DRYCC_DEFAULT_CONFIG_TAGS', '')
+KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS = int(
+    os.environ.get('KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', 2))
 
 DRYCC_APP_STORAGE_CLASS = os.environ.get('DRYCC_APP_STORAGE_CLASS', '')
 
@@ -62,3 +69,67 @@ def __getitem__(self, item):
 # WORKFLOW_MANAGER_URL = "http://127.0.0.1:8000"
 WORKFLOW_MANAGER_ACCESS_KEY = "1234567890"
 WORKFLOW_MANAGER_SECRET_KEY = "1234567890"
+
+DRYCC_VOLUME_CLAIM_TEMPLATE = """
+{% if type == 'csi' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "usage"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "{{storage_class}}",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeMode": "Filesystem"
+  }
+}
+{% elif type == 'nfs' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% elif type == 'nfs' %}
+{
+  "metadata": {
+    "annotations": {
+      "billing.drycc.cc/type": "basic"
+    }
+  },
+  "spec": {
+    "accessModes": [
+      "ReadWriteMany"
+    ],
+    "storageClassName": "",
+    "resources": {
+      "requests": {
+        "storage": "{{size}}"
+      }
+    },
+    "volumeName": "{{volume_name}}"
+  }
+}
+{% endif %}
+"""
diff --git a/rootfs/api/signals.py b/rootfs/api/signals.py
index 10cd7ca73..3b7e3831b 100644
--- a/rootfs/api/signals.py
+++ b/rootfs/api/signals.py
@@ -5,19 +5,17 @@
 """
 import time
 import hashlib
+
 import hmac
 import logging
 import urllib.parse
 import requests
-from datetime import timedelta
 from django.conf import settings
 from django.db.models.signals import post_delete, post_save
-from django.utils.timezone import now
 from django.dispatch import receiver
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
-from api.utils import get_session
-from api.tasks import retrieve_resource, send_measurements
+from api.utils import get_httpclient
+from api.tasks import send_usage
 from api.models.app import App
 from api.models.service import Service
 from api.models.gateway import Gateway
@@ -30,6 +28,7 @@
 from api.models.tls import TLS
 from api.models.volume import Volume
 from api.models.resource import Resource
+from api.models.workspace import WorkspaceInvitation
 
 
 User = get_user_model()
@@ -40,7 +39,9 @@ def _log_instance_created(**kwargs):
     if kwargs.get('created'):
         instance = kwargs['instance']
         message = '{} {} created'.format(instance.__class__.__name__.lower(), instance)
-        if hasattr(instance, 'app'):
+        if hasattr(instance, 'log'):
+            instance.log(message)
+        elif hasattr(instance, 'app'):
             instance.app.log(message)
         else:
             logger.info(message)
@@ -50,7 +51,9 @@ def _log_instance_added(**kwargs):
     if kwargs.get('created'):
         instance = kwargs['instance']
         message = '{} {} added'.format(instance.__class__.__name__.lower(), instance)
-        if hasattr(instance, 'app'):
+        if hasattr(instance, 'log'):
+            instance.log(message)
+        elif hasattr(instance, 'app'):
             instance.app.log(message)
         else:
             logger.info(message)
@@ -59,7 +62,9 @@ def _log_instance_added(**kwargs):
 def _log_instance_updated(**kwargs):
     instance = kwargs['instance']
     message = '{} {} updated'.format(instance.__class__.__name__.lower(), instance)
-    if hasattr(instance, 'app'):
+    if hasattr(instance, 'log'):
+        instance.log(message)
+    elif hasattr(instance, 'app'):
         instance.app.log(message)
     else:
         logger.info(message)
@@ -68,7 +73,9 @@ def _log_instance_updated(**kwargs):
 def _log_instance_removed(**kwargs):
     instance = kwargs['instance']
     message = '{} {} removed'.format(instance.__class__.__name__.lower(), instance)
-    if hasattr(instance, 'app'):
+    if hasattr(instance, 'log'):
+        instance.log(message)
+    elif hasattr(instance, 'app'):
         instance.app.log(message)
     else:
         logger.info(message)
@@ -79,16 +86,16 @@ def _hook_release_created(**kwargs):
     if kwargs.get('created'):
         release = kwargs['instance']
         # append release lifecycle logs to the app
-        release.app.log(release.summary)
+        release.log(release.summary)
 
         for deploy_hook in settings.DRYCC_DEPLOY_HOOK_URLS:
             url = deploy_hook
             params = {
                 'app': release.app,
-                'release': 'v{}'.format(release.version),
+                'release': release.version_name,
                 'release_summary': release.summary,
                 'sha': '',
-                'user': release.owner,
+                'user': release.app.workspace.name,
             }
             if release.build is not None:
                 params['sha'] = release.build.sha
@@ -106,11 +113,12 @@ def _hook_release_created(**kwargs):
                 ).hexdigest()
 
             try:
-                get_session().post(url, headers=headers)
+                with get_httpclient() as session:
+                    session.post(url, headers=headers)
                 # just notify with the base URL, disregard the added URL query
-                release.app.log('Deploy hook sent to {}'.format(deploy_hook))
+                release.log('Sent deploy hook to {}'.format(deploy_hook))
             except requests.RequestException as e:
-                release.app.log('An error occurred while sending the deploy hook to {}: {}'.format(
+                release.log('An error occurred while sending the deploy hook to {}: {}'.format(
                     deploy_hook, e), logging.ERROR)
 
 
@@ -131,21 +139,14 @@ def _hook_release_created(**kwargs):
 post_delete.connect(_log_instance_removed, sender=Resource, dispatch_uid='api.models.log')
 
 
-# automatically generate a new token on creation
-@receiver(post_save, sender=User)
-def create_auth_token_handle(sender, instance, created=False, **kwargs):
-    if created:
-        Token.objects.create(user=instance)
-
-
 @receiver(post_save, sender=App)
 def app_changed_handle(sender, instance: App, created=False, update_fields=None, **kwargs):
     # measure limits to workflow manager
     if settings.WORKFLOW_MANAGER_URL and not created and (
             update_fields is not None and "structure" in update_fields):
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.to_usages(timestamp), ],
         )
 
 
@@ -159,9 +160,22 @@ def tls_changed_handle(sender, instance: TLS, created=False, update_fields=None,
     if (update_fields and "certs_auto_enabled" in update_fields) or created:
         instance.refresh_certificate_to_k8s()
         for gateway in instance.app.gateway_set.all():
-            gateway.refresh_to_k8s()
+            if gateway.change_default_tls():
+                gateway.save()
+            else:
+                gateway.refresh_to_k8s()
         for route in instance.app.route_set.all():
-            route.refresh_to_k8s()
+            if route.change_default_tls():
+                route.save()
+            else:
+                route.refresh_to_k8s()
+
+
+@receiver(post_save, sender=User)
+def user_changed_handle(sender, instance: User, created=False, update_fields=None, **kwargs):
+    if created or (update_fields is not None and "email" in update_fields):
+        for invitation in WorkspaceInvitation.objects.filter(email=instance.email, accepted=True):
+            invitation.accept()
 
 
 @receiver(post_save, sender=Gateway)
@@ -178,52 +192,47 @@ def gateway_changed_handle(
 def service_changed_handle(
         sender, instance: Service, created=False, update_fields=None, **kwargs):
     if kwargs['signal'] == post_delete:
-        instance.app.route_set.filter(procfile_type=instance.procfile_type).delete()
+        for route in instance.app.route_set.all():
+            if route.cleaned_rules:
+                route.save()
+            else:
+                route.delete()
 
 
 @receiver(signal=[post_save, post_delete], sender=Domain)
 def domain_changed_handle(
         sender, instance: Domain, created=False, update_fields=None, **kwargs):
     for gateway in instance.app.gateway_set.all():
-        gateway.refresh_to_k8s()
+        if gateway.change_default_tls():
+            gateway.save()
+        else:
+            gateway.refresh_to_k8s()
     for route in instance.app.route_set.all():
-        route.refresh_to_k8s()
+        if route.change_default_tls():
+            route.save()
+        else:
+            route.refresh_to_k8s()
 
 
 @receiver(signal=[post_save, post_delete], sender=AppSettings)
 def appsettings_changed_handle(
         sender, instance: AppSettings, created=False, update_fields=None, **kwargs):
-    prev_settings, action, canaries = instance.diff_canaries()
-    if prev_settings is not None:
-        release = instance.app.release_set.filter(failed=False).latest()
-        if release.canary:
-            if action == "append":
-                instance.app.deploy(release)
-            elif action == "remove":
-                instance.app.cleanup_old()
-        for procfile_type in canaries:
-            canary = (action == "append")
-            service = instance.app.service_set.filter(procfile_type=procfile_type).first()
-            if service is not None and service.canary != canary:
-                service.canary = canary
-                service.save()
-        if prev_settings.routable != instance.routable:
-            for route in instance.app.route_set.all():
-                if route.routable != instance.routable:
-                    route.routable = instance.routable
-                    route.save()
+    prev_settings = instance.previous()
+    if prev_settings and prev_settings.routable != instance.routable:
+        for route in instance.app.route_set.all():
+            if route.routable != instance.routable:
+                route.routable = instance.routable
+                route.save()
 
 
 @receiver(post_save, sender=Config)
 def config_changed_handle(sender, instance: Config, created=False, update_fields=None, **kwargs):
     # measure limits to workflow manager
     if settings.WORKFLOW_MANAGER_URL and (
-        created or (
-            update_fields is not None and (
-                "cpu" in update_fields or "memory" in update_fields))):
+            created or (update_fields is not None and "limits" in update_fields)):
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.app.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.app.to_usages(timestamp), ],
         )
 
 
@@ -232,21 +241,14 @@ def volume_changed_handle(sender, instance: Volume, created=False, update_fields
     # measure volumes to workflow manager
     if settings.WORKFLOW_MANAGER_URL and created:
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.to_usages(timestamp), ],
         )
 
 
 @receiver(post_save, sender=Resource)
 def resource_changed_handle(
         sender, instance: Resource, created=False, update_fields=None, **kwargs):
-    # retrieve_resource
-    if created or instance.binding == "Binding" or (
-            update_fields is not None and "plan" in update_fields):
-        retrieve_resource.apply_async(
-            args=(instance, ),
-            eta=now() + timedelta(seconds=30)
-        )
     # measure resources to workflow manager
     if settings.WORKFLOW_MANAGER_URL and (
         created or (
@@ -254,6 +256,6 @@ def resource_changed_handle(
                 "plan" in update_fields
             ))):
         timestamp = time.time()
-        send_measurements.apply_async(
-            args=[instance.to_measurements(timestamp), ],
+        send_usage.apply_async(
+            args=[instance.to_usages(timestamp), ],
         )
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
index b3e536003..75242e79a 100644
--- a/rootfs/api/tasks.py
+++ b/rootfs/api/tasks.py
@@ -1,53 +1,40 @@
 # Create your tasks here
-import time
 import uuid
 import logging
 from typing import List, Dict
 from django.core import signals
 from celery import shared_task
 
-from api import manager, models
+from api import utils, manager
 from api.exceptions import ServiceUnavailable
-from api.models.resource import Resource
 logger = logging.getLogger(__name__)
 
 
 @shared_task(
-    retry_kwargs={'max_retries': None}
+    autoretry_for=(Exception, ),
+    retry_backoff=8,
+    retry_jitter=True,
+    retry_backoff_max=3600,
+    retry_kwargs={'max_retries': 32}
 )
-def retrieve_resource(resource):
+def send_usage(usage: List[Dict[str, str]]):
     task_id = uuid.uuid4().hex
     signals.request_started.send(sender=task_id)
     try:
-        if not resource.retrieve():
-            t = time.time() - resource.created.timestamp()
-            if t < 3600:
-                retrieve_resource.apply_async(args=(resource, ), countdown=30)
-            elif t < 3600 * 12:
-                retrieve_resource.apply_async(args=(resource, ), countdown=1800)
-    except (Exception, Resource.DoesNotExist) as e:
+        manager.UsageAPI().post(usage)
+    except Exception as e:
         signals.got_request_exception.send(sender=task_id)
-        if isinstance(e, Resource.DoesNotExist):
-            logger.exception("retrieve task not found resource: {}".format(resource.id))
-        else:
-            raise e
+        raise e
     else:
         signals.request_finished.send(sender=task_id)
 
 
-@shared_task(
-    autoretry_for=(Exception, ),
-    retry_backoff=8,
-    retry_jitter=True,
-    retry_backoff_max=3600,
-    retry_kwargs={'max_retries': None}
-)
-def send_measurements(measurements: List[Dict[str, str]]):
+@shared_task(retry_kwargs={'max_retries': 3})
+def send_app_log(app_id, msg, level=logging.INFO):
     task_id = uuid.uuid4().hex
     signals.request_started.send(sender=task_id)
     try:
-        measurement = manager.Measurement()
-        measurement.post(measurements)
+        utils.send_app_log(app_id, msg, level)
     except Exception as e:
         signals.got_request_exception.send(sender=task_id)
         raise e
@@ -57,7 +44,6 @@ def send_measurements(measurements: List[Dict[str, str]]):
 
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
-    retry_jitter=True,
     retry_kwargs={'max_retries': 3}
 )
 def scale_app(app, user, structure):
@@ -72,6 +58,22 @@ def scale_app(app, user, structure):
         signals.request_finished.send(sender=task_id)
 
 
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_kwargs={'max_retries': 3}
+)
+def run_pipeline(release, *args, **kwargs):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        release.app.pipeline(release, *args, **kwargs)
+    except Exception as e:
+        signals.got_request_exception.send(sender=task_id)
+        raise e
+    else:
+        signals.request_finished.send(sender=task_id)
+
+
 @shared_task(
     autoretry_for=(ServiceUnavailable, ),
     retry_jitter=True,
@@ -94,12 +96,11 @@ def restart_app(app, **kwargs):
     retry_jitter=True,
     retry_kwargs={'max_retries': 3}
 )
-def mount_app(app, user, volume):
+def delete_pod(app, **kwargs):
     task_id = uuid.uuid4().hex
     signals.request_started.send(sender=task_id)
     try:
-        app.mount(user, volume)
-        volume.save()
+        app.delete_pod(**kwargs)
     except Exception as e:
         signals.got_request_exception.send(sender=task_id)
         raise e
@@ -108,20 +109,44 @@ def mount_app(app, user, volume):
 
 
 @shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_jitter=True,
+    retry_kwargs={'max_retries': 3}
+)
+def mount_app(app, user, volume, path):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        # merge mount volume path and remove keys if a null value is provided
+        for key, value in path.items():
+            if value is None:
+                if key not in volume.path:
+                    continue
+                volume.path.pop(key)
+            else:
+                volume.path[key] = value
+        volume.save()
+        structure = {}
+        for scale_type, replicas in app.structure.items():
+            if scale_type in path:
+                structure[scale_type] = replicas
+        app.mount(user, volume, structure)
+    except Exception as e:
+        signals.got_request_exception.send(sender=task_id)
+        logger.exception(e)
+    else:
+        signals.request_finished.send(sender=task_id)
+
+
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
     retry_kwargs={'max_retries': 3}
 )
-def downstream_model_owner(app, old_owner, new_owner):
+def scale_resources(blocklist, app, suspended_state, scale_type):
     task_id = uuid.uuid4().hex
     signals.request_started.send(sender=task_id)
     try:
-        for downstream_model in [
-                models.appsettings.AppSettings, models.build.Build, models.config.Config,
-                models.domain.Domain, models.release.Release, models.resource.Resource,
-                models.tls.TLS, models.service.Service, models.volume.Volume,
-                models.gateway.Gateway, models.gateway.Route]:
-            downstream_model.objects.filter(owner=old_owner, app=app).update(owner=new_owner)
-        app.owner = new_owner
-        app.save()
+        blocklist.scale_resources(app, suspended_state, scale_type)
     except Exception as e:
         signals.got_request_exception.send(sender=task_id)
         raise e
diff --git a/rootfs/api/templates/workspace/workspace_invitation.html b/rootfs/api/templates/workspace/workspace_invitation.html
new file mode 100644
index 000000000..fd680435b
--- /dev/null
+++ b/rootfs/api/templates/workspace/workspace_invitation.html
@@ -0,0 +1,7 @@
+{% autoescape off %}
+Hi there,
+
+Please click on the link below to confirm joining {{ invitation.workspace.name }} workspace:
+
+{{ domain }}{% url 'workspace_invitation_detail' name=invitation.workspace.name uid=invitation.token %}
+{% endautoescape %}
\ No newline at end of file
diff --git a/rootfs/api/templates/workspace/workspace_invitation_accept.html b/rootfs/api/templates/workspace/workspace_invitation_accept.html
new file mode 100644
index 000000000..94c5c1893
--- /dev/null
+++ b/rootfs/api/templates/workspace/workspace_invitation_accept.html
@@ -0,0 +1,107 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>Workspace Invitation</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style>
+        :root {
+            --vscode-foreground: #CCCCCC;
+            --vscode-editor-background: #1F1F1F;
+            --vscode-textLink-foreground: #4daafc;
+        }
+        @media (prefers-color-scheme: light) {
+            :root {
+                --vscode-foreground: #3B3B3B;
+                --vscode-editor-background: #FFFFFF;
+                --vscode-textLink-foreground: #005FB8;
+            }
+        }
+        html { height: 100%; }
+        body {
+            box-sizing: border-box;
+            min-height: 100%;
+            margin: 0;
+            padding: 15px 30px;
+            display: flex;
+            flex-direction: column;
+            color: var(--vscode-foreground);
+            font-family: -apple-system, BlinkMacSystemFont, "Segoe WPC", "Segoe UI", system-ui, "Ubuntu", "Droid Sans", sans-serif;
+            background-color: var(--vscode-editor-background);
+        }
+        a { color: var(--vscode-textLink-foreground); text-decoration: none; }
+        a:hover, a:focus { text-decoration: underline; }
+        .container {
+            height: 100vh;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            text-align: center;
+        }
+        .title {
+            font-size: 28px;
+            font-weight: 300;
+            margin: 0 0 12px 0;
+            color: var(--vscode-foreground);
+        }
+        .subtitle {
+            font-size: 18px;
+            font-weight: 300;
+            color: var(--vscode-foreground);
+            opacity: 0.7;
+            margin: 0 0 36px 0;
+        }
+        .detail {
+            font-size: 14px;
+            color: var(--vscode-foreground);
+            opacity: 0.7;
+            margin: 0 0 8px 0;
+        }
+        .icon-container {
+            margin-bottom: 24px;
+        }
+        @keyframes rise-and-glow {
+            0% { opacity: 0; transform: translateY(15px) scale(0.95); }
+            50% { opacity: 0.8; transform: translateY(-2px) scale(1.02); }
+            70% { opacity: 1; transform: translateY(1px) scale(0.99); }
+            100% { opacity: 1; transform: translateY(0) scale(1); }
+        }
+        .drycc-icon {
+            width: 128px;
+            height: 128px;
+            animation: rise-and-glow 1.8s cubic-bezier(0.25, 0.46, 0.45, 0.94) forwards;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="content">
+            <div class="icon-container">
+                <svg class="drycc-icon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16">
+                    <g stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="3" cy="3" r="2"/>
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="3" cy="13" r="2"/>
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="13" cy="13" r="2"/>
+                        <circle stroke="#FF2F57" fill="#FF2F57" cx="13" cy="3" r="2"/>
+                        <rect stroke="#3FB1E5" fill="#3FB1E5" x="6.5" y="6.5" width="3" height="3"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" points="8 11 10 15 6 15"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" transform="translate(3,8) rotate(90) translate(-3,-8)" points="3 6 5 10 1 10"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" transform="translate(8,3) rotate(180) translate(-8,-3)" points="8 1 10 5 6 5"/>
+                        <polygon stroke="#38BB4C" fill="#38BB4C" stroke-linejoin="round" transform="translate(13,8) rotate(270) translate(-13,-8)" points="13 6 15 10 11 10"/>
+                    </g>
+                </svg>
+            </div>
+            {% if user_exists %}
+            <h1 class="title">Invitation Accepted</h1>
+            <p class="subtitle">You have been added to workspace <strong>{{ workspace_name }}</strong>.</p>
+            <p class="detail">Use the Drycc CLI to log in: <code>drycc login</code></p>
+            {% else %}
+            <h1 class="title">You're Invited to <strong>{{ workspace_name }}</strong></h1>
+            <p class="subtitle">You need an account to join this workspace.</p>
+            <p class="detail">First, <a href="{{ register_url }}">sign up on Drycc Passport</a>.</p>
+            <p class="detail">Then use the Drycc CLI to log in: <code>drycc login</code></p>
+            {% endif %}
+        </div>
+    </div>
+</body>
+</html>
diff --git a/rootfs/api/tests/__init__.py b/rootfs/api/tests/__init__.py
index 63df485fc..052cfe63c 100644
--- a/rootfs/api/tests/__init__.py
+++ b/rootfs/api/tests/__init__.py
@@ -2,11 +2,18 @@
 import random
 import requests_mock
 import time
+import unittest
 from os.path import dirname, realpath
 
 from django.test.runner import DiscoverRunner
+from django.contrib.auth import get_user_model
 from rest_framework.test import APITestCase, APITransactionTestCase
 
+from api.models.base import Token
+from api.models.workspace import Workspace, WorkspaceMember
+
+User = get_user_model()
+
 
 # Mock out router requests and add in some jitter
 # Used for application is available in router checks
@@ -43,34 +50,97 @@ def fake_responses(request, context):
 class SilentDjangoTestSuiteRunner(DiscoverRunner):
     """Prevents api log messages from cluttering the console during tests."""
 
-    def run_tests(self, test_labels, extra_tests=None, **kwargs):
+    def run_tests(self, test_labels, **kwargs):
         """Run tests with all but critical log messages disabled."""
         # hide any log messages less than critical
         logging.disable(logging.ERROR)
         return super(SilentDjangoTestSuiteRunner, self).run_tests(
-            test_labels, extra_tests, **kwargs)
-
-
-class DryccTestCase(APITestCase):
-
-    def create_app(self, name=None):
-        body = {}
+            test_labels, **kwargs)
+
+
+class DryccBaseTestCase(unittest.TestCase):
+
+    def _get_authenticated_user(self):
+        credentials = getattr(getattr(self, 'client', None), '_credentials', {})
+        auth = credentials.get('HTTP_AUTHORIZATION', '')
+        if auth.startswith('Token '):
+            token_key = auth.split(' ', 1)[1]
+            token = Token.objects.filter(key=token_key).select_related('owner').first()
+            if token is not None:
+                return token.owner
+        return getattr(self, 'user', None)
+
+    def _default_workspace_name(self):
+        user = self._get_authenticated_user()
+        if user and user.username:
+            base = ''.join(ch for ch in user.username.lower() if ch.isalnum())
+            if len(base) >= 5:
+                return base
+        return 'autotest'
+
+    def _ensure_workspace_admin(self, workspace_name):
+        user = self._get_authenticated_user()
+        if user is None:
+            user = User.objects.filter(username='autotest').first()
+        if user is None:
+            raise AssertionError('No test user available for workspace membership')
+
+        workspace, _ = Workspace.objects.get_or_create(
+            name=workspace_name,
+            defaults={'email': user.email or f'{workspace_name}@example.com'},
+        )
+        WorkspaceMember.objects.update_or_create(
+            workspace=workspace,
+            user=user,
+            defaults={'role': 'admin'},
+        )
+        return workspace.name
+
+    def create_app(self, name=None, workspace=None):
+        workspace_name = workspace or self._default_workspace_name()
+        workspace_name = self._ensure_workspace_admin(workspace_name)
+
+        body = {'workspace': workspace_name}
         if name:
-            body = {'id': name}
+            body['id'] = name
 
         response = self.client.post('/v2/apps', body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertIn('id', response.data)
         return response.data['id']
 
-
-class DryccTransactionTestCase(APITransactionTestCase):
-    def create_app(self, name=None):
-        body = {}
-        if name:
-            body = {'id': name}
-
-        response = self.client.post('/v2/apps', body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('id', response.data)
-        return response.data['id']
+    def get_or_create_token(self, user):
+        token, _ = Token.objects.get_or_create(
+            owner=user,
+            defaults={
+                "oauth": {
+                    "access_token": "test",
+                    "expires_in": 3600 * 24 * 7,
+                    "token_type": "Bearer",
+                    "scope": "openid",
+                    "refresh_token": "test",
+                }
+            }
+        )
+        return token.key
+
+    def assertPodContains(self, pods, app_id, ptype, version, state="up"):
+        for pod in pods:
+            if (pod["type"] == ptype and
+                    pod["release"] == version and pod["state"] == state):
+                pod_name = app_id + '-%s-[0-9]{1,10}-[a-z0-9]{5}' % ptype
+                self.assertRegex(pod['name'], pod_name)
+                return
+        raise ValueError(
+            "pod not contains: ptype={}, version={}, state={}".format(
+                ptype, version, state
+            )
+        )
+
+
+class DryccTransactionTestCase(DryccBaseTestCase, APITransactionTestCase):
+    pass
+
+
+class DryccTestCase(DryccBaseTestCase, APITestCase):
+    pass
diff --git a/rootfs/api/tests/admissions/deployment_scale.json b/rootfs/api/tests/admissions/deployment_scale.json
new file mode 100644
index 000000000..af6e049ca
--- /dev/null
+++ b/rootfs/api/tests/admissions/deployment_scale.json
@@ -0,0 +1,102 @@
+{
+    "kind": "AdmissionReview",
+    "apiVersion": "admission.k8s.io/v1",
+    "request": {
+        "uid": "2c7ac31e-fd85-4674-8a22-a720299ff0a9",
+        "kind": {
+            "group": "autoscaling",
+            "version": "v1",
+            "kind": "Scale"
+        },
+        "resource": {
+            "group": "apps",
+            "version": "v1",
+            "resource": "deployments"
+        },
+        "subResource": "scale",
+        "requestKind": {
+            "group": "autoscaling",
+            "version": "v1",
+            "kind": "Scale"
+        },
+        "requestResource": {
+            "group": "apps",
+            "version": "v1",
+            "resource": "deployments"
+        },
+        "requestSubResource": "scale",
+        "name": "myapp-web",
+        "namespace": "myapp",
+        "operation": "UPDATE",
+        "userInfo": {
+            "username": "system:admin",
+            "groups": ["system:masters", "system:authenticated"]
+        },
+        "object": {
+            "kind": "Scale",
+            "apiVersion": "autoscaling/v1",
+            "metadata": {
+                "name": "myapp-web",
+                "namespace": "myapp",
+                "uid": "302b2d9a-06b3-4d2b-af86-c6f8cc7c13cb",
+                "resourceVersion": "61988667",
+                "creationTimestamp": "2023-12-12T03:40:51Z",
+                "managedFields": [{
+                    "manager": "kubectl",
+                    "operation": "Update",
+                    "apiVersion": "autoscaling/v1",
+                    "time": "2024-03-04T04:47:33Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:spec": {
+                            "f:replicas": {}
+                        }
+                    },
+                    "subresource": "scale"
+                }]
+            },
+            "spec": {
+                "replicas": 2
+            },
+            "status": {
+                "replicas": 3,
+                "selector": "app=myapp,heritage=drycc,type=web"
+            }
+        },
+        "oldObject": {
+            "kind": "Scale",
+            "apiVersion": "autoscaling/v1",
+            "metadata": {
+                "name": "myapp-web",
+                "namespace": "myapp",
+                "uid": "302b2d9a-06b3-4d2b-af86-c6f8cc7c13cb",
+                "resourceVersion": "61988667",
+                "creationTimestamp": "2023-12-12T03:40:51Z",
+                "managedFields": [{
+                    "manager": "kubectl",
+                    "operation": "Update",
+                    "apiVersion": "autoscaling/v1",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:spec": {
+                            "f:replicas": {}
+                        }
+                    },
+                    "subresource": "scale"
+                }]
+            },
+            "spec": {
+                "replicas": 3
+            },
+            "status": {
+                "replicas": 3,
+                "selector": "app=myapp,heritage=drycc,type=web"
+            }
+        },
+        "dryRun": false,
+        "options": {
+            "kind": "UpdateOptions",
+            "apiVersion": "meta.k8s.io/v1"
+        }
+    }
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/admissions/job_status_create_ok.json b/rootfs/api/tests/admissions/job_status_create_ok.json
new file mode 100644
index 000000000..214ee0c18
--- /dev/null
+++ b/rootfs/api/tests/admissions/job_status_create_ok.json
@@ -0,0 +1,854 @@
+{
+    "kind": "AdmissionReview",
+    "apiVersion": "admission.k8s.io/v1",
+    "request": {
+        "uid": "24ffe91b-b66f-4e7e-80ca-1501ebad53bb",
+        "kind": {
+            "group": "batch",
+            "version": "v1",
+            "kind": "Job"
+        },
+        "resource": {
+            "group": "batch",
+            "version": "v1",
+            "resource": "jobs"
+        },
+        "subResource": "status",
+        "requestKind": {
+            "group": "batch",
+            "version": "v1",
+            "kind": "Job"
+        },
+        "requestResource": {
+            "group": "batch",
+            "version": "v1",
+            "resource": "jobs"
+        },
+        "requestSubResource": "status",
+        "name": "myapp-run-5cp7y",
+        "namespace": "myapp",
+        "operation": "UPDATE",
+        "userInfo": {
+            "username": "system:serviceaccount:kube-system:job-controller",
+            "uid": "b119efc6-5e7b-41d6-91b5-38fa11b3971d",
+            "groups": ["system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"]
+        },
+        "object": {
+            "kind": "Job",
+            "apiVersion": "batch/v1",
+            "metadata": {
+                "name": "myapp-run-5cp7y",
+                "namespace": "myapp",
+                "uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                "resourceVersion": "62050800",
+                "generation": 1,
+                "creationTimestamp": "2024-03-04T04:42:24Z",
+                "labels": {
+                    "app": "myapp",
+                    "heritage": "drycc",
+                    "type": "run"
+                },
+                "managedFields": [{
+                    "manager": "Drycc Controller",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:42:24Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:metadata": {
+                            "f:labels": {
+                                ".": {},
+                                "f:app": {},
+                                "f:heritage": {},
+                                "f:type": {}
+                            }
+                        },
+                        "f:spec": {
+                            "f:activeDeadlineSeconds": {},
+                            "f:backoffLimit": {},
+                            "f:completionMode": {},
+                            "f:completions": {},
+                            "f:parallelism": {},
+                            "f:suspend": {},
+                            "f:template": {
+                                "f:metadata": {
+                                    "f:annotations": {
+                                        ".": {},
+                                        "f:kubernetes.io/egress-bandwidth": {},
+                                        "f:kubernetes.io/ingress-bandwidth": {}
+                                    },
+                                    "f:labels": {
+                                        ".": {},
+                                        "f:app": {},
+                                        "f:heritage": {},
+                                        "f:type": {},
+                                        "f:version": {}
+                                    },
+                                    "f:name": {},
+                                    "f:namespace": {}
+                                },
+                                "f:spec": {
+                                    "f:affinity": {
+                                        ".": {},
+                                        "f:podAntiAffinity": {
+                                            ".": {},
+                                            "f:preferredDuringSchedulingIgnoredDuringExecution": {}
+                                        }
+                                    },
+                                    "f:containers": {
+                                        "k:{\"name\":\"myapp-run\"}": {
+                                            ".": {},
+                                            "f:args": {},
+                                            "f:env": {
+                                                ".": {},
+                                                "k:{\"name\":\"DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DISABLE_COLLECTSTATIC\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_APP\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:value": {}
+                                                },
+                                                "k:{\"name\":\"IMAGE_PULL_POLICY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PIP_INDEX_URL\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PORT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"SOURCE_VERSION\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_CREATED_AT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_SUMMARY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"a\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                }
+                                            },
+                                            "f:image": {},
+                                            "f:imagePullPolicy": {},
+                                            "f:name": {},
+                                            "f:resources": {
+                                                ".": {},
+                                                "f:limits": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                },
+                                                "f:requests": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                }
+                                            },
+                                            "f:securityContext": {
+                                                ".": {},
+                                                "f:allowPrivilegeEscalation": {},
+                                                "f:capabilities": {
+                                                    ".": {},
+                                                    "f:add": {}
+                                                }
+                                            },
+                                            "f:terminationMessagePath": {},
+                                            "f:terminationMessagePolicy": {}
+                                        }
+                                    },
+                                    "f:dnsPolicy": {},
+                                    "f:imagePullSecrets": {
+                                        ".": {},
+                                        "k:{\"name\":\"private-registry\"}": {}
+                                    },
+                                    "f:restartPolicy": {},
+                                    "f:runtimeClassName": {},
+                                    "f:schedulerName": {},
+                                    "f:securityContext": {},
+                                    "f:terminationGracePeriodSeconds": {}
+                                }
+                            },
+                            "f:ttlSecondsAfterFinished": {}
+                        }
+                    }
+                }, {
+                    "manager": "k3s",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:42:24Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:status": {
+                            "f:active": {},
+                            "f:ready": {},
+                            "f:startTime": {},
+                            "f:uncountedTerminatedPods": {}
+                        }
+                    },
+                    "subresource": "status"
+                }]
+            },
+            "spec": {
+                "parallelism": 1,
+                "completions": 1,
+                "activeDeadlineSeconds": 3600,
+                "backoffLimit": 0,
+                "selector": {
+                    "matchLabels": {
+                        "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "name": "myapp-run-5cp7y",
+                        "namespace": "myapp",
+                        "creationTimestamp": null,
+                        "labels": {
+                            "app": "myapp",
+                            "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "batch.kubernetes.io/job-name": "myapp-run-5cp7y",
+                            "controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "heritage": "drycc",
+                            "job-name": "myapp-run-5cp7y",
+                            "type": "run",
+                            "version": "v11"
+                        },
+                        "annotations": {
+                            "kubernetes.io/egress-bandwidth": "100M",
+                            "kubernetes.io/ingress-bandwidth": "100M"
+                        }
+                    },
+                    "spec": {
+                        "containers": [{
+                            "name": "myapp-run",
+                            "image": "registry.uucin.com/lijianguo/py3django3:vim",
+                            "args": ["sleep", "30s"],
+                            "env": [{
+                                "name": "DEBUG",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "debug"
+                                    }
+                                }
+                            }, {
+                                "name": "DISABLE_COLLECTSTATIC",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "disable-collectstatic"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_APP",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "drycc-app"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_DEBUG",
+                                "value": "1"
+                            }, {
+                                "name": "IMAGE_PULL_POLICY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "image-pull-policy"
+                                    }
+                                }
+                            }, {
+                                "name": "PIP_INDEX_URL",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "pip-index-url"
+                                    }
+                                }
+                            }, {
+                                "name": "PORT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "port"
+                                    }
+                                }
+                            }, {
+                                "name": "SOURCE_VERSION",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "source-version"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_CREATED_AT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-created-at"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_SUMMARY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-summary"
+                                    }
+                                }
+                            }, {
+                                "name": "a",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "a"
+                                    }
+                                }
+                            }],
+                            "resources": {
+                                "limits": {
+                                    "ephemeral-storage": "2Gi"
+                                },
+                                "requests": {
+                                    "ephemeral-storage": "1Gi"
+                                }
+                            },
+                            "terminationMessagePath": "/dev/termination-log",
+                            "terminationMessagePolicy": "File",
+                            "imagePullPolicy": "Always",
+                            "securityContext": {
+                                "capabilities": {
+                                    "add": ["SYS_ADMIN"]
+                                },
+                                "allowPrivilegeEscalation": true
+                            }
+                        }],
+                        "restartPolicy": "Never",
+                        "terminationGracePeriodSeconds": 30,
+                        "dnsPolicy": "ClusterFirst",
+                        "securityContext": {},
+                        "imagePullSecrets": [{
+                            "name": "private-registry"
+                        }],
+                        "affinity": {
+                            "podAntiAffinity": {
+                                "preferredDuringSchedulingIgnoredDuringExecution": [{
+                                    "weight": 100,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "topology.kubernetes.io/zone"
+                                    }
+                                }, {
+                                    "weight": 90,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "kubernetes.io/hostname"
+                                    }
+                                }]
+                            }
+                        },
+                        "schedulerName": "default-scheduler",
+                        "runtimeClassName": "runsc"
+                    }
+                },
+                "ttlSecondsAfterFinished": 86400,
+                "completionMode": "NonIndexed",
+                "suspend": false
+            },
+            "status": {
+                "startTime": "2024-03-04T04:42:24Z",
+                "active": 1,
+                "uncountedTerminatedPods": {},
+                "ready": 0
+            }
+        },
+        "oldObject": {
+            "kind": "Job",
+            "apiVersion": "batch/v1",
+            "metadata": {
+                "name": "myapp-run-5cp7y",
+                "namespace": "myapp",
+                "uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                "resourceVersion": "62050800",
+                "generation": 1,
+                "creationTimestamp": "2024-03-04T04:42:24Z",
+                "labels": {
+                    "app": "myapp",
+                    "heritage": "drycc",
+                    "type": "run"
+                },
+                "managedFields": [{
+                    "manager": "Drycc Controller",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:42:24Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:metadata": {
+                            "f:labels": {
+                                ".": {},
+                                "f:app": {},
+                                "f:heritage": {},
+                                "f:type": {}
+                            }
+                        },
+                        "f:spec": {
+                            "f:activeDeadlineSeconds": {},
+                            "f:backoffLimit": {},
+                            "f:completionMode": {},
+                            "f:completions": {},
+                            "f:parallelism": {},
+                            "f:suspend": {},
+                            "f:template": {
+                                "f:metadata": {
+                                    "f:annotations": {
+                                        ".": {},
+                                        "f:kubernetes.io/egress-bandwidth": {},
+                                        "f:kubernetes.io/ingress-bandwidth": {}
+                                    },
+                                    "f:labels": {
+                                        ".": {},
+                                        "f:app": {},
+                                        "f:heritage": {},
+                                        "f:type": {},
+                                        "f:version": {}
+                                    },
+                                    "f:name": {},
+                                    "f:namespace": {}
+                                },
+                                "f:spec": {
+                                    "f:affinity": {
+                                        ".": {},
+                                        "f:podAntiAffinity": {
+                                            ".": {},
+                                            "f:preferredDuringSchedulingIgnoredDuringExecution": {}
+                                        }
+                                    },
+                                    "f:containers": {
+                                        "k:{\"name\":\"myapp-run\"}": {
+                                            ".": {},
+                                            "f:args": {},
+                                            "f:env": {
+                                                ".": {},
+                                                "k:{\"name\":\"DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DISABLE_COLLECTSTATIC\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_APP\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:value": {}
+                                                },
+                                                "k:{\"name\":\"IMAGE_PULL_POLICY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PIP_INDEX_URL\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PORT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"SOURCE_VERSION\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_CREATED_AT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_SUMMARY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"a\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                }
+                                            },
+                                            "f:image": {},
+                                            "f:imagePullPolicy": {},
+                                            "f:name": {},
+                                            "f:resources": {
+                                                ".": {},
+                                                "f:limits": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                },
+                                                "f:requests": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                }
+                                            },
+                                            "f:securityContext": {
+                                                ".": {},
+                                                "f:allowPrivilegeEscalation": {},
+                                                "f:capabilities": {
+                                                    ".": {},
+                                                    "f:add": {}
+                                                }
+                                            },
+                                            "f:terminationMessagePath": {},
+                                            "f:terminationMessagePolicy": {}
+                                        }
+                                    },
+                                    "f:dnsPolicy": {},
+                                    "f:imagePullSecrets": {
+                                        ".": {},
+                                        "k:{\"name\":\"private-registry\"}": {}
+                                    },
+                                    "f:restartPolicy": {},
+                                    "f:runtimeClassName": {},
+                                    "f:schedulerName": {},
+                                    "f:securityContext": {},
+                                    "f:terminationGracePeriodSeconds": {}
+                                }
+                            },
+                            "f:ttlSecondsAfterFinished": {}
+                        }
+                    }
+                }]
+            },
+            "spec": {
+                "parallelism": 1,
+                "completions": 1,
+                "activeDeadlineSeconds": 3600,
+                "backoffLimit": 0,
+                "selector": {
+                    "matchLabels": {
+                        "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "name": "myapp-run-5cp7y",
+                        "namespace": "myapp",
+                        "creationTimestamp": null,
+                        "labels": {
+                            "app": "myapp",
+                            "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "batch.kubernetes.io/job-name": "myapp-run-5cp7y",
+                            "controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "heritage": "drycc",
+                            "job-name": "myapp-run-5cp7y",
+                            "type": "run",
+                            "version": "v11"
+                        },
+                        "annotations": {
+                            "kubernetes.io/egress-bandwidth": "100M",
+                            "kubernetes.io/ingress-bandwidth": "100M"
+                        }
+                    },
+                    "spec": {
+                        "containers": [{
+                            "name": "myapp-run",
+                            "image": "registry.uucin.com/lijianguo/py3django3:vim",
+                            "args": ["sleep", "30s"],
+                            "env": [{
+                                "name": "DEBUG",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "debug"
+                                    }
+                                }
+                            }, {
+                                "name": "DISABLE_COLLECTSTATIC",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "disable-collectstatic"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_APP",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "drycc-app"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_DEBUG",
+                                "value": "1"
+                            }, {
+                                "name": "IMAGE_PULL_POLICY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "image-pull-policy"
+                                    }
+                                }
+                            }, {
+                                "name": "PIP_INDEX_URL",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "pip-index-url"
+                                    }
+                                }
+                            }, {
+                                "name": "PORT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "port"
+                                    }
+                                }
+                            }, {
+                                "name": "SOURCE_VERSION",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "source-version"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_CREATED_AT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-created-at"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_SUMMARY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-summary"
+                                    }
+                                }
+                            }, {
+                                "name": "a",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "a"
+                                    }
+                                }
+                            }],
+                            "resources": {
+                                "limits": {
+                                    "ephemeral-storage": "2Gi"
+                                },
+                                "requests": {
+                                    "ephemeral-storage": "1Gi"
+                                }
+                            },
+                            "terminationMessagePath": "/dev/termination-log",
+                            "terminationMessagePolicy": "File",
+                            "imagePullPolicy": "Always",
+                            "securityContext": {
+                                "capabilities": {
+                                    "add": ["SYS_ADMIN"]
+                                },
+                                "allowPrivilegeEscalation": true
+                            }
+                        }],
+                        "restartPolicy": "Never",
+                        "terminationGracePeriodSeconds": 30,
+                        "dnsPolicy": "ClusterFirst",
+                        "securityContext": {},
+                        "imagePullSecrets": [{
+                            "name": "private-registry"
+                        }],
+                        "affinity": {
+                            "podAntiAffinity": {
+                                "preferredDuringSchedulingIgnoredDuringExecution": [{
+                                    "weight": 100,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "topology.kubernetes.io/zone"
+                                    }
+                                }, {
+                                    "weight": 90,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "kubernetes.io/hostname"
+                                    }
+                                }]
+                            }
+                        },
+                        "schedulerName": "default-scheduler",
+                        "runtimeClassName": "runsc"
+                    }
+                },
+                "ttlSecondsAfterFinished": 86400,
+                "completionMode": "NonIndexed",
+                "suspend": false
+            },
+            "status": {}
+        },
+        "dryRun": false,
+        "options": {
+            "kind": "UpdateOptions",
+            "apiVersion": "meta.k8s.io/v1"
+        }
+    }
+}
diff --git a/rootfs/api/tests/admissions/job_status_failed.json b/rootfs/api/tests/admissions/job_status_failed.json
new file mode 100644
index 000000000..ca4f44588
--- /dev/null
+++ b/rootfs/api/tests/admissions/job_status_failed.json
@@ -0,0 +1,889 @@
+{
+    "kind": "AdmissionReview",
+    "apiVersion": "admission.k8s.io/v1",
+    "request": {
+        "uid": "c334b9cd-7468-4a19-a002-15d59db75b90",
+        "kind": {
+            "group": "batch",
+            "version": "v1",
+            "kind": "Job"
+        },
+        "resource": {
+            "group": "batch",
+            "version": "v1",
+            "resource": "jobs"
+        },
+        "subResource": "status",
+        "requestKind": {
+            "group": "batch",
+            "version": "v1",
+            "kind": "Job"
+        },
+        "requestResource": {
+            "group": "batch",
+            "version": "v1",
+            "resource": "jobs"
+        },
+        "requestSubResource": "status",
+        "name": "myapp-run-53gm6",
+        "namespace": "myapp",
+        "operation": "UPDATE",
+        "userInfo": {
+            "username": "system:serviceaccount:kube-system:job-controller",
+            "uid": "b119efc6-5e7b-41d6-91b5-38fa11b3971d",
+            "groups": ["system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"]
+        },
+        "object": {
+            "kind": "Job",
+            "apiVersion": "batch/v1",
+            "metadata": {
+                "name": "myapp-run-53gm6",
+                "namespace": "myapp",
+                "uid": "40c98ecb-330e-4167-b02c-db5f276cecec",
+                "resourceVersion": "62052324",
+                "generation": 1,
+                "creationTimestamp": "2024-03-04T04:45:17Z",
+                "labels": {
+                    "app": "myapp",
+                    "heritage": "drycc",
+                    "type": "run"
+                },
+                "managedFields": [{
+                    "manager": "Drycc Controller",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:45:17Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:metadata": {
+                            "f:labels": {
+                                ".": {},
+                                "f:app": {},
+                                "f:heritage": {},
+                                "f:type": {}
+                            }
+                        },
+                        "f:spec": {
+                            "f:activeDeadlineSeconds": {},
+                            "f:backoffLimit": {},
+                            "f:completionMode": {},
+                            "f:completions": {},
+                            "f:parallelism": {},
+                            "f:suspend": {},
+                            "f:template": {
+                                "f:metadata": {
+                                    "f:annotations": {
+                                        ".": {},
+                                        "f:kubernetes.io/egress-bandwidth": {},
+                                        "f:kubernetes.io/ingress-bandwidth": {}
+                                    },
+                                    "f:labels": {
+                                        ".": {},
+                                        "f:app": {},
+                                        "f:heritage": {},
+                                        "f:type": {},
+                                        "f:version": {}
+                                    },
+                                    "f:name": {},
+                                    "f:namespace": {}
+                                },
+                                "f:spec": {
+                                    "f:affinity": {
+                                        ".": {},
+                                        "f:podAntiAffinity": {
+                                            ".": {},
+                                            "f:preferredDuringSchedulingIgnoredDuringExecution": {}
+                                        }
+                                    },
+                                    "f:containers": {
+                                        "k:{\"name\":\"myapp-run\"}": {
+                                            ".": {},
+                                            "f:args": {},
+                                            "f:env": {
+                                                ".": {},
+                                                "k:{\"name\":\"DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DISABLE_COLLECTSTATIC\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_APP\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:value": {}
+                                                },
+                                                "k:{\"name\":\"IMAGE_PULL_POLICY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PIP_INDEX_URL\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PORT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"SOURCE_VERSION\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_CREATED_AT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_SUMMARY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"a\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                }
+                                            },
+                                            "f:image": {},
+                                            "f:imagePullPolicy": {},
+                                            "f:name": {},
+                                            "f:resources": {
+                                                ".": {},
+                                                "f:limits": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                },
+                                                "f:requests": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                }
+                                            },
+                                            "f:securityContext": {
+                                                ".": {},
+                                                "f:allowPrivilegeEscalation": {},
+                                                "f:capabilities": {
+                                                    ".": {},
+                                                    "f:add": {}
+                                                }
+                                            },
+                                            "f:terminationMessagePath": {},
+                                            "f:terminationMessagePolicy": {}
+                                        }
+                                    },
+                                    "f:dnsPolicy": {},
+                                    "f:imagePullSecrets": {
+                                        ".": {},
+                                        "k:{\"name\":\"private-registry\"}": {}
+                                    },
+                                    "f:restartPolicy": {},
+                                    "f:runtimeClassName": {},
+                                    "f:schedulerName": {},
+                                    "f:securityContext": {},
+                                    "f:terminationGracePeriodSeconds": {}
+                                }
+                            },
+                            "f:ttlSecondsAfterFinished": {}
+                        }
+                    }
+                }, {
+                    "manager": "k3s",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:45:27Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:status": {
+                            "f:conditions": {},
+                            "f:failed": {},
+                            "f:ready": {},
+                            "f:startTime": {},
+                            "f:uncountedTerminatedPods": {}
+                        }
+                    },
+                    "subresource": "status"
+                }]
+            },
+            "spec": {
+                "parallelism": 1,
+                "completions": 1,
+                "activeDeadlineSeconds": 3600,
+                "backoffLimit": 0,
+                "selector": {
+                    "matchLabels": {
+                        "batch.kubernetes.io/controller-uid": "40c98ecb-330e-4167-b02c-db5f276cecec"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "name": "myapp-run-53gm6",
+                        "namespace": "myapp",
+                        "creationTimestamp": null,
+                        "labels": {
+                            "app": "myapp",
+                            "batch.kubernetes.io/controller-uid": "40c98ecb-330e-4167-b02c-db5f276cecec",
+                            "batch.kubernetes.io/job-name": "myapp-run-53gm6",
+                            "controller-uid": "40c98ecb-330e-4167-b02c-db5f276cecec",
+                            "heritage": "drycc",
+                            "job-name": "myapp-run-53gm6",
+                            "type": "run",
+                            "version": "v11"
+                        },
+                        "annotations": {
+                            "kubernetes.io/egress-bandwidth": "100M",
+                            "kubernetes.io/ingress-bandwidth": "100M"
+                        }
+                    },
+                    "spec": {
+                        "containers": [{
+                            "name": "myapp-run",
+                            "image": "registry.uucin.com/lijianguo/py3django3:vim",
+                            "args": ["amex2"],
+                            "env": [{
+                                "name": "DEBUG",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "debug"
+                                    }
+                                }
+                            }, {
+                                "name": "DISABLE_COLLECTSTATIC",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "disable-collectstatic"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_APP",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "drycc-app"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_DEBUG",
+                                "value": "1"
+                            }, {
+                                "name": "IMAGE_PULL_POLICY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "image-pull-policy"
+                                    }
+                                }
+                            }, {
+                                "name": "PIP_INDEX_URL",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "pip-index-url"
+                                    }
+                                }
+                            }, {
+                                "name": "PORT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "port"
+                                    }
+                                }
+                            }, {
+                                "name": "SOURCE_VERSION",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "source-version"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_CREATED_AT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-created-at"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_SUMMARY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-summary"
+                                    }
+                                }
+                            }, {
+                                "name": "a",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "a"
+                                    }
+                                }
+                            }],
+                            "resources": {
+                                "limits": {
+                                    "ephemeral-storage": "2Gi"
+                                },
+                                "requests": {
+                                    "ephemeral-storage": "1Gi"
+                                }
+                            },
+                            "terminationMessagePath": "/dev/termination-log",
+                            "terminationMessagePolicy": "File",
+                            "imagePullPolicy": "Always",
+                            "securityContext": {
+                                "capabilities": {
+                                    "add": ["SYS_ADMIN"]
+                                },
+                                "allowPrivilegeEscalation": true
+                            }
+                        }],
+                        "restartPolicy": "Never",
+                        "terminationGracePeriodSeconds": 30,
+                        "dnsPolicy": "ClusterFirst",
+                        "securityContext": {},
+                        "imagePullSecrets": [{
+                            "name": "private-registry"
+                        }],
+                        "affinity": {
+                            "podAntiAffinity": {
+                                "preferredDuringSchedulingIgnoredDuringExecution": [{
+                                    "weight": 100,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-53gm6"]
+                                            }]
+                                        },
+                                        "topologyKey": "topology.kubernetes.io/zone"
+                                    }
+                                }, {
+                                    "weight": 90,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-53gm6"]
+                                            }]
+                                        },
+                                        "topologyKey": "kubernetes.io/hostname"
+                                    }
+                                }]
+                            }
+                        },
+                        "schedulerName": "default-scheduler",
+                        "runtimeClassName": "runsc"
+                    }
+                },
+                "ttlSecondsAfterFinished": 86400,
+                "completionMode": "NonIndexed",
+                "suspend": false
+            },
+            "status": {
+                "conditions": [{
+                    "type": "Failed",
+                    "status": "True",
+                    "lastProbeTime": "2024-03-04T04:45:27Z",
+                    "lastTransitionTime": "2024-03-04T04:45:27Z",
+                    "reason": "BackoffLimitExceeded",
+                    "message": "Job has reached the specified backoff limit"
+                }],
+                "startTime": "2024-03-04T04:45:17Z",
+                "failed": 1,
+                "uncountedTerminatedPods": {},
+                "ready": 0
+            }
+        },
+        "oldObject": {
+            "kind": "Job",
+            "apiVersion": "batch/v1",
+            "metadata": {
+                "name": "myapp-run-53gm6",
+                "namespace": "myapp",
+                "uid": "40c98ecb-330e-4167-b02c-db5f276cecec",
+                "resourceVersion": "62052324",
+                "generation": 1,
+                "creationTimestamp": "2024-03-04T04:45:17Z",
+                "labels": {
+                    "app": "myapp",
+                    "heritage": "drycc",
+                    "type": "run"
+                },
+                "managedFields": [{
+                    "manager": "Drycc Controller",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:45:17Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:metadata": {
+                            "f:labels": {
+                                ".": {},
+                                "f:app": {},
+                                "f:heritage": {},
+                                "f:type": {}
+                            }
+                        },
+                        "f:spec": {
+                            "f:activeDeadlineSeconds": {},
+                            "f:backoffLimit": {},
+                            "f:completionMode": {},
+                            "f:completions": {},
+                            "f:parallelism": {},
+                            "f:suspend": {},
+                            "f:template": {
+                                "f:metadata": {
+                                    "f:annotations": {
+                                        ".": {},
+                                        "f:kubernetes.io/egress-bandwidth": {},
+                                        "f:kubernetes.io/ingress-bandwidth": {}
+                                    },
+                                    "f:labels": {
+                                        ".": {},
+                                        "f:app": {},
+                                        "f:heritage": {},
+                                        "f:type": {},
+                                        "f:version": {}
+                                    },
+                                    "f:name": {},
+                                    "f:namespace": {}
+                                },
+                                "f:spec": {
+                                    "f:affinity": {
+                                        ".": {},
+                                        "f:podAntiAffinity": {
+                                            ".": {},
+                                            "f:preferredDuringSchedulingIgnoredDuringExecution": {}
+                                        }
+                                    },
+                                    "f:containers": {
+                                        "k:{\"name\":\"myapp-run\"}": {
+                                            ".": {},
+                                            "f:args": {},
+                                            "f:env": {
+                                                ".": {},
+                                                "k:{\"name\":\"DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DISABLE_COLLECTSTATIC\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_APP\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:value": {}
+                                                },
+                                                "k:{\"name\":\"IMAGE_PULL_POLICY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PIP_INDEX_URL\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PORT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"SOURCE_VERSION\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_CREATED_AT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_SUMMARY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"a\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                }
+                                            },
+                                            "f:image": {},
+                                            "f:imagePullPolicy": {},
+                                            "f:name": {},
+                                            "f:resources": {
+                                                ".": {},
+                                                "f:limits": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                },
+                                                "f:requests": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                }
+                                            },
+                                            "f:securityContext": {
+                                                ".": {},
+                                                "f:allowPrivilegeEscalation": {},
+                                                "f:capabilities": {
+                                                    ".": {},
+                                                    "f:add": {}
+                                                }
+                                            },
+                                            "f:terminationMessagePath": {},
+                                            "f:terminationMessagePolicy": {}
+                                        }
+                                    },
+                                    "f:dnsPolicy": {},
+                                    "f:imagePullSecrets": {
+                                        ".": {},
+                                        "k:{\"name\":\"private-registry\"}": {}
+                                    },
+                                    "f:restartPolicy": {},
+                                    "f:runtimeClassName": {},
+                                    "f:schedulerName": {},
+                                    "f:securityContext": {},
+                                    "f:terminationGracePeriodSeconds": {}
+                                }
+                            },
+                            "f:ttlSecondsAfterFinished": {}
+                        }
+                    }
+                }, {
+                    "manager": "k3s",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:45:27Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:status": {
+                            "f:ready": {},
+                            "f:startTime": {},
+                            "f:uncountedTerminatedPods": {
+                                ".": {},
+                                "f:failed": {
+                                    ".": {},
+                                    "v:\"9910fb43-847b-4452-a4cb-f035c7fff664\"": {}
+                                }
+                            }
+                        }
+                    },
+                    "subresource": "status"
+                }]
+            },
+            "spec": {
+                "parallelism": 1,
+                "completions": 1,
+                "activeDeadlineSeconds": 3600,
+                "backoffLimit": 0,
+                "selector": {
+                    "matchLabels": {
+                        "batch.kubernetes.io/controller-uid": "40c98ecb-330e-4167-b02c-db5f276cecec"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "name": "myapp-run-53gm6",
+                        "namespace": "myapp",
+                        "creationTimestamp": null,
+                        "labels": {
+                            "app": "myapp",
+                            "batch.kubernetes.io/controller-uid": "40c98ecb-330e-4167-b02c-db5f276cecec",
+                            "batch.kubernetes.io/job-name": "myapp-run-53gm6",
+                            "controller-uid": "40c98ecb-330e-4167-b02c-db5f276cecec",
+                            "heritage": "drycc",
+                            "job-name": "myapp-run-53gm6",
+                            "type": "run",
+                            "version": "v11"
+                        },
+                        "annotations": {
+                            "kubernetes.io/egress-bandwidth": "100M",
+                            "kubernetes.io/ingress-bandwidth": "100M"
+                        }
+                    },
+                    "spec": {
+                        "containers": [{
+                            "name": "myapp-run",
+                            "image": "registry.uucin.com/lijianguo/py3django3:vim",
+                            "args": ["amex2"],
+                            "env": [{
+                                "name": "DEBUG",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "debug"
+                                    }
+                                }
+                            }, {
+                                "name": "DISABLE_COLLECTSTATIC",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "disable-collectstatic"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_APP",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "drycc-app"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_DEBUG",
+                                "value": "1"
+                            }, {
+                                "name": "IMAGE_PULL_POLICY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "image-pull-policy"
+                                    }
+                                }
+                            }, {
+                                "name": "PIP_INDEX_URL",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "pip-index-url"
+                                    }
+                                }
+                            }, {
+                                "name": "PORT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "port"
+                                    }
+                                }
+                            }, {
+                                "name": "SOURCE_VERSION",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "source-version"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_CREATED_AT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-created-at"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_SUMMARY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-summary"
+                                    }
+                                }
+                            }, {
+                                "name": "a",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "a"
+                                    }
+                                }
+                            }],
+                            "resources": {
+                                "limits": {
+                                    "ephemeral-storage": "2Gi"
+                                },
+                                "requests": {
+                                    "ephemeral-storage": "1Gi"
+                                }
+                            },
+                            "terminationMessagePath": "/dev/termination-log",
+                            "terminationMessagePolicy": "File",
+                            "imagePullPolicy": "Always",
+                            "securityContext": {
+                                "capabilities": {
+                                    "add": ["SYS_ADMIN"]
+                                },
+                                "allowPrivilegeEscalation": true
+                            }
+                        }],
+                        "restartPolicy": "Never",
+                        "terminationGracePeriodSeconds": 30,
+                        "dnsPolicy": "ClusterFirst",
+                        "securityContext": {},
+                        "imagePullSecrets": [{
+                            "name": "private-registry"
+                        }],
+                        "affinity": {
+                            "podAntiAffinity": {
+                                "preferredDuringSchedulingIgnoredDuringExecution": [{
+                                    "weight": 100,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-53gm6"]
+                                            }]
+                                        },
+                                        "topologyKey": "topology.kubernetes.io/zone"
+                                    }
+                                }, {
+                                    "weight": 90,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-53gm6"]
+                                            }]
+                                        },
+                                        "topologyKey": "kubernetes.io/hostname"
+                                    }
+                                }]
+                            }
+                        },
+                        "schedulerName": "default-scheduler",
+                        "runtimeClassName": "runsc"
+                    }
+                },
+                "ttlSecondsAfterFinished": 86400,
+                "completionMode": "NonIndexed",
+                "suspend": false
+            },
+            "status": {
+                "startTime": "2024-03-04T04:45:17Z",
+                "uncountedTerminatedPods": {
+                    "failed": ["9910fb43-847b-4452-a4cb-f035c7fff664"]
+                },
+                "ready": 0
+            }
+        },
+        "dryRun": false,
+        "options": {
+            "kind": "UpdateOptions",
+            "apiVersion": "meta.k8s.io/v1"
+        }
+    }
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/admissions/job_status_succeeded.json b/rootfs/api/tests/admissions/job_status_succeeded.json
new file mode 100644
index 000000000..ca8596f73
--- /dev/null
+++ b/rootfs/api/tests/admissions/job_status_succeeded.json
@@ -0,0 +1,889 @@
+{
+    "kind": "AdmissionReview",
+    "apiVersion": "admission.k8s.io/v1",
+    "request": {
+        "uid": "469f28d2-8689-4d92-8c27-dcab78389952",
+        "kind": {
+            "group": "batch",
+            "version": "v1",
+            "kind": "Job"
+        },
+        "resource": {
+            "group": "batch",
+            "version": "v1",
+            "resource": "jobs"
+        },
+        "subResource": "status",
+        "requestKind": {
+            "group": "batch",
+            "version": "v1",
+            "kind": "Job"
+        },
+        "requestResource": {
+            "group": "batch",
+            "version": "v1",
+            "resource": "jobs"
+        },
+        "requestSubResource": "status",
+        "name": "myapp-run-5cp7y",
+        "namespace": "myapp",
+        "operation": "UPDATE",
+        "userInfo": {
+            "username": "system:serviceaccount:kube-system:job-controller",
+            "uid": "b119efc6-5e7b-41d6-91b5-38fa11b3971d",
+            "groups": ["system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"]
+        },
+        "object": {
+            "kind": "Job",
+            "apiVersion": "batch/v1",
+            "metadata": {
+                "name": "myapp-run-5cp7y",
+                "namespace": "myapp",
+                "uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                "resourceVersion": "62051112",
+                "generation": 1,
+                "creationTimestamp": "2024-03-04T04:42:24Z",
+                "labels": {
+                    "app": "myapp",
+                    "heritage": "drycc",
+                    "type": "run"
+                },
+                "managedFields": [{
+                    "manager": "Drycc Controller",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:42:24Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:metadata": {
+                            "f:labels": {
+                                ".": {},
+                                "f:app": {},
+                                "f:heritage": {},
+                                "f:type": {}
+                            }
+                        },
+                        "f:spec": {
+                            "f:activeDeadlineSeconds": {},
+                            "f:backoffLimit": {},
+                            "f:completionMode": {},
+                            "f:completions": {},
+                            "f:parallelism": {},
+                            "f:suspend": {},
+                            "f:template": {
+                                "f:metadata": {
+                                    "f:annotations": {
+                                        ".": {},
+                                        "f:kubernetes.io/egress-bandwidth": {},
+                                        "f:kubernetes.io/ingress-bandwidth": {}
+                                    },
+                                    "f:labels": {
+                                        ".": {},
+                                        "f:app": {},
+                                        "f:heritage": {},
+                                        "f:type": {},
+                                        "f:version": {}
+                                    },
+                                    "f:name": {},
+                                    "f:namespace": {}
+                                },
+                                "f:spec": {
+                                    "f:affinity": {
+                                        ".": {},
+                                        "f:podAntiAffinity": {
+                                            ".": {},
+                                            "f:preferredDuringSchedulingIgnoredDuringExecution": {}
+                                        }
+                                    },
+                                    "f:containers": {
+                                        "k:{\"name\":\"myapp-run\"}": {
+                                            ".": {},
+                                            "f:args": {},
+                                            "f:env": {
+                                                ".": {},
+                                                "k:{\"name\":\"DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DISABLE_COLLECTSTATIC\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_APP\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:value": {}
+                                                },
+                                                "k:{\"name\":\"IMAGE_PULL_POLICY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PIP_INDEX_URL\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PORT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"SOURCE_VERSION\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_CREATED_AT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_SUMMARY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"a\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                }
+                                            },
+                                            "f:image": {},
+                                            "f:imagePullPolicy": {},
+                                            "f:name": {},
+                                            "f:resources": {
+                                                ".": {},
+                                                "f:limits": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                },
+                                                "f:requests": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                }
+                                            },
+                                            "f:securityContext": {
+                                                ".": {},
+                                                "f:allowPrivilegeEscalation": {},
+                                                "f:capabilities": {
+                                                    ".": {},
+                                                    "f:add": {}
+                                                }
+                                            },
+                                            "f:terminationMessagePath": {},
+                                            "f:terminationMessagePolicy": {}
+                                        }
+                                    },
+                                    "f:dnsPolicy": {},
+                                    "f:imagePullSecrets": {
+                                        ".": {},
+                                        "k:{\"name\":\"private-registry\"}": {}
+                                    },
+                                    "f:restartPolicy": {},
+                                    "f:runtimeClassName": {},
+                                    "f:schedulerName": {},
+                                    "f:securityContext": {},
+                                    "f:terminationGracePeriodSeconds": {}
+                                }
+                            },
+                            "f:ttlSecondsAfterFinished": {}
+                        }
+                    }
+                }, {
+                    "manager": "k3s",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:43:01Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:status": {
+                            "f:completionTime": {},
+                            "f:conditions": {},
+                            "f:ready": {},
+                            "f:startTime": {},
+                            "f:succeeded": {},
+                            "f:uncountedTerminatedPods": {}
+                        }
+                    },
+                    "subresource": "status"
+                }]
+            },
+            "spec": {
+                "parallelism": 1,
+                "completions": 1,
+                "activeDeadlineSeconds": 3600,
+                "backoffLimit": 0,
+                "selector": {
+                    "matchLabels": {
+                        "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "name": "myapp-run-5cp7y",
+                        "namespace": "myapp",
+                        "creationTimestamp": null,
+                        "labels": {
+                            "app": "myapp",
+                            "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "batch.kubernetes.io/job-name": "myapp-run-5cp7y",
+                            "controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "heritage": "drycc",
+                            "job-name": "myapp-run-5cp7y",
+                            "type": "run",
+                            "version": "v11"
+                        },
+                        "annotations": {
+                            "kubernetes.io/egress-bandwidth": "100M",
+                            "kubernetes.io/ingress-bandwidth": "100M"
+                        }
+                    },
+                    "spec": {
+                        "containers": [{
+                            "name": "myapp-run",
+                            "image": "registry.uucin.com/lijianguo/py3django3:vim",
+                            "args": ["sleep", "30s"],
+                            "env": [{
+                                "name": "DEBUG",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "debug"
+                                    }
+                                }
+                            }, {
+                                "name": "DISABLE_COLLECTSTATIC",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "disable-collectstatic"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_APP",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "drycc-app"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_DEBUG",
+                                "value": "1"
+                            }, {
+                                "name": "IMAGE_PULL_POLICY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "image-pull-policy"
+                                    }
+                                }
+                            }, {
+                                "name": "PIP_INDEX_URL",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "pip-index-url"
+                                    }
+                                }
+                            }, {
+                                "name": "PORT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "port"
+                                    }
+                                }
+                            }, {
+                                "name": "SOURCE_VERSION",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "source-version"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_CREATED_AT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-created-at"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_SUMMARY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-summary"
+                                    }
+                                }
+                            }, {
+                                "name": "a",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "a"
+                                    }
+                                }
+                            }],
+                            "resources": {
+                                "limits": {
+                                    "ephemeral-storage": "2Gi"
+                                },
+                                "requests": {
+                                    "ephemeral-storage": "1Gi"
+                                }
+                            },
+                            "terminationMessagePath": "/dev/termination-log",
+                            "terminationMessagePolicy": "File",
+                            "imagePullPolicy": "Always",
+                            "securityContext": {
+                                "capabilities": {
+                                    "add": ["SYS_ADMIN"]
+                                },
+                                "allowPrivilegeEscalation": true
+                            }
+                        }],
+                        "restartPolicy": "Never",
+                        "terminationGracePeriodSeconds": 30,
+                        "dnsPolicy": "ClusterFirst",
+                        "securityContext": {},
+                        "imagePullSecrets": [{
+                            "name": "private-registry"
+                        }],
+                        "affinity": {
+                            "podAntiAffinity": {
+                                "preferredDuringSchedulingIgnoredDuringExecution": [{
+                                    "weight": 100,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "topology.kubernetes.io/zone"
+                                    }
+                                }, {
+                                    "weight": 90,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "kubernetes.io/hostname"
+                                    }
+                                }]
+                            }
+                        },
+                        "schedulerName": "default-scheduler",
+                        "runtimeClassName": "runsc"
+                    }
+                },
+                "ttlSecondsAfterFinished": 86400,
+                "completionMode": "NonIndexed",
+                "suspend": false
+            },
+            "status": {
+                "conditions": [{
+                    "type": "Complete",
+                    "status": "True",
+                    "lastProbeTime": "2024-03-04T04:43:01Z",
+                    "lastTransitionTime": "2024-03-04T04:43:01Z"
+                }],
+                "startTime": "2024-03-04T04:42:24Z",
+                "completionTime": "2024-03-04T04:43:01Z",
+                "succeeded": 1,
+                "uncountedTerminatedPods": {},
+                "ready": 0
+            }
+        },
+        "oldObject": {
+            "kind": "Job",
+            "apiVersion": "batch/v1",
+            "metadata": {
+                "name": "myapp-run-5cp7y",
+                "namespace": "myapp",
+                "uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                "resourceVersion": "62051112",
+                "generation": 1,
+                "creationTimestamp": "2024-03-04T04:42:24Z",
+                "labels": {
+                    "app": "myapp",
+                    "heritage": "drycc",
+                    "type": "run"
+                },
+                "managedFields": [{
+                    "manager": "Drycc Controller",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:42:24Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:metadata": {
+                            "f:labels": {
+                                ".": {},
+                                "f:app": {},
+                                "f:heritage": {},
+                                "f:type": {}
+                            }
+                        },
+                        "f:spec": {
+                            "f:activeDeadlineSeconds": {},
+                            "f:backoffLimit": {},
+                            "f:completionMode": {},
+                            "f:completions": {},
+                            "f:parallelism": {},
+                            "f:suspend": {},
+                            "f:template": {
+                                "f:metadata": {
+                                    "f:annotations": {
+                                        ".": {},
+                                        "f:kubernetes.io/egress-bandwidth": {},
+                                        "f:kubernetes.io/ingress-bandwidth": {}
+                                    },
+                                    "f:labels": {
+                                        ".": {},
+                                        "f:app": {},
+                                        "f:heritage": {},
+                                        "f:type": {},
+                                        "f:version": {}
+                                    },
+                                    "f:name": {},
+                                    "f:namespace": {}
+                                },
+                                "f:spec": {
+                                    "f:affinity": {
+                                        ".": {},
+                                        "f:podAntiAffinity": {
+                                            ".": {},
+                                            "f:preferredDuringSchedulingIgnoredDuringExecution": {}
+                                        }
+                                    },
+                                    "f:containers": {
+                                        "k:{\"name\":\"myapp-run\"}": {
+                                            ".": {},
+                                            "f:args": {},
+                                            "f:env": {
+                                                ".": {},
+                                                "k:{\"name\":\"DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DISABLE_COLLECTSTATIC\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_APP\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"DRYCC_DEBUG\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:value": {}
+                                                },
+                                                "k:{\"name\":\"IMAGE_PULL_POLICY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PIP_INDEX_URL\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"PORT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"SOURCE_VERSION\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_CREATED_AT\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"WORKFLOW_RELEASE_SUMMARY\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                },
+                                                "k:{\"name\":\"a\"}": {
+                                                    ".": {},
+                                                    "f:name": {},
+                                                    "f:valueFrom": {
+                                                        ".": {},
+                                                        "f:secretKeyRef": {}
+                                                    }
+                                                }
+                                            },
+                                            "f:image": {},
+                                            "f:imagePullPolicy": {},
+                                            "f:name": {},
+                                            "f:resources": {
+                                                ".": {},
+                                                "f:limits": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                },
+                                                "f:requests": {
+                                                    ".": {},
+                                                    "f:ephemeral-storage": {}
+                                                }
+                                            },
+                                            "f:securityContext": {
+                                                ".": {},
+                                                "f:allowPrivilegeEscalation": {},
+                                                "f:capabilities": {
+                                                    ".": {},
+                                                    "f:add": {}
+                                                }
+                                            },
+                                            "f:terminationMessagePath": {},
+                                            "f:terminationMessagePolicy": {}
+                                        }
+                                    },
+                                    "f:dnsPolicy": {},
+                                    "f:imagePullSecrets": {
+                                        ".": {},
+                                        "k:{\"name\":\"private-registry\"}": {}
+                                    },
+                                    "f:restartPolicy": {},
+                                    "f:runtimeClassName": {},
+                                    "f:schedulerName": {},
+                                    "f:securityContext": {},
+                                    "f:terminationGracePeriodSeconds": {}
+                                }
+                            },
+                            "f:ttlSecondsAfterFinished": {}
+                        }
+                    }
+                }, {
+                    "manager": "k3s",
+                    "operation": "Update",
+                    "apiVersion": "batch/v1",
+                    "time": "2024-03-04T04:43:01Z",
+                    "fieldsType": "FieldsV1",
+                    "fieldsV1": {
+                        "f:status": {
+                            "f:ready": {},
+                            "f:startTime": {},
+                            "f:uncountedTerminatedPods": {
+                                ".": {},
+                                "f:succeeded": {
+                                    ".": {},
+                                    "v:\"215306ac-d875-44c6-be71-890bc2da6811\"": {}
+                                }
+                            }
+                        }
+                    },
+                    "subresource": "status"
+                }]
+            },
+            "spec": {
+                "parallelism": 1,
+                "completions": 1,
+                "activeDeadlineSeconds": 3600,
+                "backoffLimit": 0,
+                "selector": {
+                    "matchLabels": {
+                        "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "name": "myapp-run-5cp7y",
+                        "namespace": "myapp",
+                        "creationTimestamp": null,
+                        "labels": {
+                            "app": "myapp",
+                            "batch.kubernetes.io/controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "batch.kubernetes.io/job-name": "myapp-run-5cp7y",
+                            "controller-uid": "02a0d19d-ad83-4063-9124-b9d7d7d8d359",
+                            "heritage": "drycc",
+                            "job-name": "myapp-run-5cp7y",
+                            "type": "run",
+                            "version": "v11"
+                        },
+                        "annotations": {
+                            "kubernetes.io/egress-bandwidth": "100M",
+                            "kubernetes.io/ingress-bandwidth": "100M"
+                        }
+                    },
+                    "spec": {
+                        "containers": [{
+                            "name": "myapp-run",
+                            "image": "registry.uucin.com/lijianguo/py3django3:vim",
+                            "args": ["sleep", "30s"],
+                            "env": [{
+                                "name": "DEBUG",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "debug"
+                                    }
+                                }
+                            }, {
+                                "name": "DISABLE_COLLECTSTATIC",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "disable-collectstatic"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_APP",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "drycc-app"
+                                    }
+                                }
+                            }, {
+                                "name": "DRYCC_DEBUG",
+                                "value": "1"
+                            }, {
+                                "name": "IMAGE_PULL_POLICY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "image-pull-policy"
+                                    }
+                                }
+                            }, {
+                                "name": "PIP_INDEX_URL",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "pip-index-url"
+                                    }
+                                }
+                            }, {
+                                "name": "PORT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "port"
+                                    }
+                                }
+                            }, {
+                                "name": "SOURCE_VERSION",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "source-version"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_CREATED_AT",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-created-at"
+                                    }
+                                }
+                            }, {
+                                "name": "WORKFLOW_RELEASE_SUMMARY",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "workflow-release-summary"
+                                    }
+                                }
+                            }, {
+                                "name": "a",
+                                "valueFrom": {
+                                    "secretKeyRef": {
+                                        "name": "myapp-v11-env",
+                                        "key": "a"
+                                    }
+                                }
+                            }],
+                            "resources": {
+                                "limits": {
+                                    "ephemeral-storage": "2Gi"
+                                },
+                                "requests": {
+                                    "ephemeral-storage": "1Gi"
+                                }
+                            },
+                            "terminationMessagePath": "/dev/termination-log",
+                            "terminationMessagePolicy": "File",
+                            "imagePullPolicy": "Always",
+                            "securityContext": {
+                                "capabilities": {
+                                    "add": ["SYS_ADMIN"]
+                                },
+                                "allowPrivilegeEscalation": true
+                            }
+                        }],
+                        "restartPolicy": "Never",
+                        "terminationGracePeriodSeconds": 30,
+                        "dnsPolicy": "ClusterFirst",
+                        "securityContext": {},
+                        "imagePullSecrets": [{
+                            "name": "private-registry"
+                        }],
+                        "affinity": {
+                            "podAntiAffinity": {
+                                "preferredDuringSchedulingIgnoredDuringExecution": [{
+                                    "weight": 100,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "topology.kubernetes.io/zone"
+                                    }
+                                }, {
+                                    "weight": 90,
+                                    "podAffinityTerm": {
+                                        "labelSelector": {
+                                            "matchExpressions": [{
+                                                "key": "app",
+                                                "operator": "In",
+                                                "values": ["myapp-run-5cp7y"]
+                                            }]
+                                        },
+                                        "topologyKey": "kubernetes.io/hostname"
+                                    }
+                                }]
+                            }
+                        },
+                        "schedulerName": "default-scheduler",
+                        "runtimeClassName": "runsc"
+                    }
+                },
+                "ttlSecondsAfterFinished": 86400,
+                "completionMode": "NonIndexed",
+                "suspend": false
+            },
+            "status": {
+                "startTime": "2024-03-04T04:42:24Z",
+                "uncountedTerminatedPods": {
+                    "succeeded": ["215306ac-d875-44c6-be71-890bc2da6811"]
+                },
+                "ready": 0
+            }
+        },
+        "dryRun": false,
+        "options": {
+            "kind": "UpdateOptions",
+            "apiVersion": "meta.k8s.io/v1"
+        }
+    }
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json b/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json
new file mode 100644
index 000000000..5dc9641cc
--- /dev/null
+++ b/rootfs/api/tests/admissions/service_binding_binding_request_in_flight.json
@@ -0,0 +1,285 @@
+{
+	"kind": "AdmissionReview",
+	"apiVersion": "admission.k8s.io/v1",
+	"request": {
+		"uid": "16d0a538-0414-459d-b548-4b2dc4ed791f",
+		"kind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceBinding"
+		},
+		"resource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "servicebindings"
+		},
+		"subResource": "status",
+		"requestKind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceBinding"
+		},
+		"requestResource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "servicebindings"
+		},
+		"requestSubResource": "status",
+		"name": "valkey-t2",
+		"namespace": "myapp",
+		"operation": "UPDATE",
+		"userInfo": {
+			"username": "system:serviceaccount:catalog:service-catalog-controller-manager",
+			"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+			"groups": [
+				"system:serviceaccounts",
+				"system:serviceaccounts:catalog",
+				"system:authenticated"
+			],
+			"extra": {
+				"authentication.kubernetes.io/pod-name": [
+					"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+				],
+				"authentication.kubernetes.io/pod-uid": [
+					"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+				]
+			}
+		},
+		"object": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceBinding",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:48:21Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 1,
+				"labels": {
+					"heritage": "drycc"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:instanceRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:48:21Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:currentOperation": {},
+								"f:inProgressProperties": {
+									".": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:operationStartTime": {},
+								"f:orphanMitigationInProgress": {},
+								"f:reconciledGeneration": {},
+								"f:unbindStatus": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:48:21Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74043277",
+				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
+			},
+			"spec": {
+				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
+				"instanceRef": {
+					"name": "valkey-t2"
+				},
+				"secretName": "valkey-t2",
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"drycc-controller-6b95d76b68-cx78l"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:drycc",
+						"system:authenticated"
+					],
+					"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+					"username": "system:serviceaccount:drycc:drycc-controller"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": false,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:48:21Z",
+						"message": "Binding request for ServiceBinding in-flight to Broker",
+						"reason": "BindingRequestInFlight",
+						"status": "False",
+						"type": "Ready"
+					}
+				],
+				"currentOperation": "Bind",
+				"inProgressProperties": {
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"drycc-controller-6b95d76b68-cx78l"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:drycc",
+							"system:authenticated"
+						],
+						"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+						"username": "system:serviceaccount:drycc:drycc-controller"
+					}
+				},
+				"lastConditionState": "BindingRequestInFlight",
+				"operationStartTime": "2024-03-25T01:48:21Z",
+				"orphanMitigationInProgress": false,
+				"reconciledGeneration": 0,
+				"unbindStatus": "Required"
+			}
+		},
+		"oldObject": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceBinding",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:48:21Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 1,
+				"labels": {
+					"heritage": "drycc"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:instanceRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:48:21Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:lastConditionState": {},
+								"f:orphanMitigationInProgress": {},
+								"f:reconciledGeneration": {},
+								"f:unbindStatus": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:48:21Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74043277",
+				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
+			},
+			"spec": {
+				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
+				"instanceRef": {
+					"name": "valkey-t2"
+				},
+				"secretName": "valkey-t2",
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"drycc-controller-6b95d76b68-cx78l"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:drycc",
+						"system:authenticated"
+					],
+					"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+					"username": "system:serviceaccount:drycc:drycc-controller"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": false,
+				"conditions": [],
+				"lastConditionState": "",
+				"orphanMitigationInProgress": false,
+				"reconciledGeneration": 0,
+				"unbindStatus": "NotRequired"
+			}
+		},
+		"dryRun": false,
+		"options": {
+			"kind": "UpdateOptions",
+			"apiVersion": "meta.k8s.io/v1"
+		}
+	}
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/admissions/service_binding_ready.json b/rootfs/api/tests/admissions/service_binding_ready.json
new file mode 100644
index 000000000..d486b302b
--- /dev/null
+++ b/rootfs/api/tests/admissions/service_binding_ready.json
@@ -0,0 +1,326 @@
+{
+	"kind": "AdmissionReview",
+	"apiVersion": "admission.k8s.io/v1",
+	"request": {
+		"uid": "19f316cc-3bec-4457-bf3a-f98153f7b2d3",
+		"kind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceBinding"
+		},
+		"resource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "servicebindings"
+		},
+		"subResource": "status",
+		"requestKind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceBinding"
+		},
+		"requestResource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "servicebindings"
+		},
+		"requestSubResource": "status",
+		"name": "valkey-t2",
+		"namespace": "myapp",
+		"operation": "UPDATE",
+		"userInfo": {
+			"username": "system:serviceaccount:catalog:service-catalog-controller-manager",
+			"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+			"groups": [
+				"system:serviceaccounts",
+				"system:serviceaccounts:catalog",
+				"system:authenticated"
+			],
+			"extra": {
+				"authentication.kubernetes.io/pod-name": [
+					"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+				],
+				"authentication.kubernetes.io/pod-uid": [
+					"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+				]
+			}
+		},
+		"object": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceBinding",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:48:21Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 1,
+				"labels": {
+					"heritage": "drycc"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:instanceRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:48:21Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:externalProperties": {
+									".": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:orphanMitigationInProgress": {},
+								"f:reconciledGeneration": {},
+								"f:unbindStatus": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:48:22Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74043278",
+				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
+			},
+			"spec": {
+				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
+				"instanceRef": {
+					"name": "valkey-t2"
+				},
+				"secretName": "valkey-t2",
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"drycc-controller-6b95d76b68-cx78l"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:drycc",
+						"system:authenticated"
+					],
+					"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+					"username": "system:serviceaccount:drycc:drycc-controller"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": false,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:48:22Z",
+						"message": "Injected bind result",
+						"reason": "InjectedBindResult",
+						"status": "True",
+						"type": "Ready"
+					}
+				],
+				"externalProperties": {
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"drycc-controller-6b95d76b68-cx78l"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:drycc",
+							"system:authenticated"
+						],
+						"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+						"username": "system:serviceaccount:drycc:drycc-controller"
+					}
+				},
+				"lastConditionState": "Ready",
+				"orphanMitigationInProgress": false,
+				"reconciledGeneration": 1,
+				"unbindStatus": "Required"
+			}
+		},
+		"oldObject": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceBinding",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:48:21Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 1,
+				"labels": {
+					"heritage": "drycc"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:instanceRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:48:21Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:currentOperation": {},
+								"f:inProgressProperties": {
+									".": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:operationStartTime": {},
+								"f:orphanMitigationInProgress": {},
+								"f:reconciledGeneration": {},
+								"f:unbindStatus": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:48:21Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74043278",
+				"uid": "292b0ef5-947f-42a8-adfb-909e99d26906"
+			},
+			"spec": {
+				"externalID": "57414c7d-0931-4310-8ed6-d8368ac0d3de",
+				"instanceRef": {
+					"name": "valkey-t2"
+				},
+				"secretName": "valkey-t2",
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"drycc-controller-6b95d76b68-cx78l"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:drycc",
+						"system:authenticated"
+					],
+					"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+					"username": "system:serviceaccount:drycc:drycc-controller"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": false,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:48:21Z",
+						"message": "Binding request for ServiceBinding in-flight to Broker",
+						"reason": "BindingRequestInFlight",
+						"status": "False",
+						"type": "Ready"
+					}
+				],
+				"currentOperation": "Bind",
+				"inProgressProperties": {
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"drycc-controller-6b95d76b68-cx78l"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"10c4fe5b-fea5-47d9-a49e-10ff94ba8e08"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:drycc",
+							"system:authenticated"
+						],
+						"uid": "13ad2917-0e39-4f88-8b2e-4d77e5797ea6",
+						"username": "system:serviceaccount:drycc:drycc-controller"
+					}
+				},
+				"lastConditionState": "BindingRequestInFlight",
+				"operationStartTime": "2024-03-25T01:48:21Z",
+				"orphanMitigationInProgress": false,
+				"reconciledGeneration": 0,
+				"unbindStatus": "Required"
+			}
+		},
+		"dryRun": false,
+		"options": {
+			"kind": "UpdateOptions",
+			"apiVersion": "meta.k8s.io/v1"
+		}
+	}
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/admissions/service_instance_provisioning.json b/rootfs/api/tests/admissions/service_instance_provisioning.json
new file mode 100644
index 000000000..fd4f68f17
--- /dev/null
+++ b/rootfs/api/tests/admissions/service_instance_provisioning.json
@@ -0,0 +1,402 @@
+{
+	"kind": "AdmissionReview",
+	"apiVersion": "admission.k8s.io/v1",
+	"request": {
+		"uid": "46e7e259-2160-4923-8b8a-b0fa6daf3b0e",
+		"kind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceInstance"
+		},
+		"resource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "serviceinstances"
+		},
+		"subResource": "status",
+		"requestKind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceInstance"
+		},
+		"requestResource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "serviceinstances"
+		},
+		"requestSubResource": "status",
+		"name": "valkey-t2",
+		"namespace": "myapp",
+		"operation": "UPDATE",
+		"userInfo": {
+			"username": "system:serviceaccount:catalog:service-catalog-controller-manager",
+			"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+			"groups": [
+				"system:serviceaccounts",
+				"system:serviceaccounts:catalog",
+				"system:authenticated"
+			],
+			"extra": {
+				"authentication.kubernetes.io/pod-name": [
+					"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+				],
+				"authentication.kubernetes.io/pod-uid": [
+					"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+				]
+			}
+		},
+		"object": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceInstance",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:34:27Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 2,
+				"labels": {
+					"heritage": "drycc",
+					"servicecatalog.k8s.io/spec.clusterServiceClassRef.name": "d1225ad72f9ea4317aac1415103d81e947e19a27c681bd02193ed278",
+					"servicecatalog.k8s.io/spec.clusterServicePlanRef.name": "c243d1d2925e51b1116b0b5d9a0d240d8847a97b2819405b483fc650"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:clusterServiceClassExternalName": {},
+								"f:clusterServicePlanExternalName": {}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:34:27Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:spec": {
+								"f:clusterServiceClassRef": {
+									".": {},
+									"f:name": {}
+								},
+								"f:clusterServicePlanRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"time": "2024-03-25T01:42:28Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:currentOperation": {},
+								"f:deprovisionStatus": {},
+								"f:inProgressProperties": {
+									".": {},
+									"f:clusterServicePlanExternalID": {},
+									"f:clusterServicePlanExternalName": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:observedGeneration": {},
+								"f:operationStartTime": {},
+								"f:orphanMitigationInProgress": {},
+								"f:provisionStatus": {},
+								"f:reconciledGeneration": {},
+								"f:userSpecifiedClassName": {},
+								"f:userSpecifiedPlanName": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:42:31Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74040375",
+				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
+			},
+			"spec": {
+				"clusterServiceClassExternalName": "valkey",
+				"clusterServiceClassRef": {
+					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
+				},
+				"clusterServicePlanExternalName": "standard-128",
+				"clusterServicePlanRef": {
+					"name": "8d659058-a3b4-4058-b039-cc03a31b9442"
+				},
+				"externalID": "f5d24838-b765-4988-8381-3f2d0945ddc1",
+				"updateRequests": 0,
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:catalog",
+						"system:authenticated"
+					],
+					"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+					"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": true,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:42:28Z",
+						"message": "The instance is being provisioned asynchronously (provision f5d24838-b765-4988-8381-3f2d0945ddc1 in progress at 1711330950.8151164)",
+						"reason": "Provisioning",
+						"status": "False",
+						"type": "Ready"
+					}
+				],
+				"currentOperation": "Provision",
+				"deprovisionStatus": "Required",
+				"inProgressProperties": {
+					"clusterServicePlanExternalID": "8d659058-a3b4-4058-b039-cc03a31b9442",
+					"clusterServicePlanExternalName": "standard-128",
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:catalog",
+							"system:authenticated"
+						],
+						"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+						"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+					}
+				},
+				"lastConditionState": "Provisioning",
+				"observedGeneration": 2,
+				"operationStartTime": "2024-03-25T01:42:28Z",
+				"orphanMitigationInProgress": false,
+				"provisionStatus": "",
+				"reconciledGeneration": 0,
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
+				"userSpecifiedPlanName": "standard-128"
+			}
+		},
+		"oldObject": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceInstance",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:34:27Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 2,
+				"labels": {
+					"heritage": "drycc",
+					"servicecatalog.k8s.io/spec.clusterServiceClassRef.name": "d1225ad72f9ea4317aac1415103d81e947e19a27c681bd02193ed278",
+					"servicecatalog.k8s.io/spec.clusterServicePlanRef.name": "c243d1d2925e51b1116b0b5d9a0d240d8847a97b2819405b483fc650"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:clusterServiceClassExternalName": {},
+								"f:clusterServicePlanExternalName": {}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:34:27Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:spec": {
+								"f:clusterServiceClassRef": {
+									".": {},
+									"f:name": {}
+								},
+								"f:clusterServicePlanRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"time": "2024-03-25T01:42:28Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:currentOperation": {},
+								"f:deprovisionStatus": {},
+								"f:inProgressProperties": {
+									".": {},
+									"f:clusterServicePlanExternalID": {},
+									"f:clusterServicePlanExternalName": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:observedGeneration": {},
+								"f:operationStartTime": {},
+								"f:orphanMitigationInProgress": {},
+								"f:provisionStatus": {},
+								"f:reconciledGeneration": {},
+								"f:userSpecifiedClassName": {},
+								"f:userSpecifiedPlanName": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:42:31Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74040375",
+				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
+			},
+			"spec": {
+				"clusterServiceClassExternalName": "valkey",
+				"clusterServiceClassRef": {
+					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
+				},
+				"clusterServicePlanExternalName": "standard-128",
+				"clusterServicePlanRef": {
+					"name": "8d659058-a3b4-4058-b039-cc03a31b9442"
+				},
+				"externalID": "f5d24838-b765-4988-8381-3f2d0945ddc1",
+				"updateRequests": 0,
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:catalog",
+						"system:authenticated"
+					],
+					"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+					"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": true,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:42:28Z",
+						"message": "The instance is being provisioned asynchronously (provision f5d24838-b765-4988-8381-3f2d0945ddc1 in progress at 1711330950.8151164)",
+						"reason": "Provisioning",
+						"status": "False",
+						"type": "Ready"
+					}
+				],
+				"currentOperation": "Provision",
+				"deprovisionStatus": "Required",
+				"inProgressProperties": {
+					"clusterServicePlanExternalID": "8d659058-a3b4-4058-b039-cc03a31b9442",
+					"clusterServicePlanExternalName": "standard-128",
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:catalog",
+							"system:authenticated"
+						],
+						"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+						"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+					}
+				},
+				"lastConditionState": "Provisioning",
+				"observedGeneration": 2,
+				"operationStartTime": "2024-03-25T01:42:28Z",
+				"orphanMitigationInProgress": false,
+				"provisionStatus": "",
+				"reconciledGeneration": 0,
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
+				"userSpecifiedPlanName": "standard-128"
+			}
+		},
+		"dryRun": false,
+		"options": {
+			"kind": "UpdateOptions",
+			"apiVersion": "meta.k8s.io/v1"
+		}
+	}
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/admissions/service_instance_ready.json b/rootfs/api/tests/admissions/service_instance_ready.json
new file mode 100644
index 000000000..60f833f3a
--- /dev/null
+++ b/rootfs/api/tests/admissions/service_instance_ready.json
@@ -0,0 +1,398 @@
+{
+	"kind": "AdmissionReview",
+	"apiVersion": "admission.k8s.io/v1",
+	"request": {
+		"uid": "f1e4340f-a8d0-4710-8e9c-80ae570456a4",
+		"kind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceInstance"
+		},
+		"resource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "serviceinstances"
+		},
+		"subResource": "status",
+		"requestKind": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"kind": "ServiceInstance"
+		},
+		"requestResource": {
+			"group": "servicecatalog.k8s.io",
+			"version": "v1beta1",
+			"resource": "serviceinstances"
+		},
+		"requestSubResource": "status",
+		"name": "valkey-t2",
+		"namespace": "myapp",
+		"operation": "UPDATE",
+		"userInfo": {
+			"username": "system:serviceaccount:catalog:service-catalog-controller-manager",
+			"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+			"groups": [
+				"system:serviceaccounts",
+				"system:serviceaccounts:catalog",
+				"system:authenticated"
+			],
+			"extra": {
+				"authentication.kubernetes.io/pod-name": [
+					"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+				],
+				"authentication.kubernetes.io/pod-uid": [
+					"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+				]
+			}
+		},
+		"object": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceInstance",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:34:27Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 2,
+				"labels": {
+					"heritage": "drycc",
+					"servicecatalog.k8s.io/spec.clusterServiceClassRef.name": "d1225ad72f9ea4317aac1415103d81e947e19a27c681bd02193ed278",
+					"servicecatalog.k8s.io/spec.clusterServicePlanRef.name": "c243d1d2925e51b1116b0b5d9a0d240d8847a97b2819405b483fc650"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:clusterServiceClassExternalName": {},
+								"f:clusterServicePlanExternalName": {}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:34:27Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:spec": {
+								"f:clusterServiceClassRef": {
+									".": {},
+									"f:name": {}
+								},
+								"f:clusterServicePlanRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"time": "2024-03-25T01:42:28Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:deprovisionStatus": {},
+								"f:externalProperties": {
+									".": {},
+									"f:clusterServicePlanExternalID": {},
+									"f:clusterServicePlanExternalName": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:observedGeneration": {},
+								"f:orphanMitigationInProgress": {},
+								"f:provisionStatus": {},
+								"f:reconciledGeneration": {},
+								"f:userSpecifiedClassName": {},
+								"f:userSpecifiedPlanName": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:44:38Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74040375",
+				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
+			},
+			"spec": {
+				"clusterServiceClassExternalName": "valkey",
+				"clusterServiceClassRef": {
+					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
+				},
+				"clusterServicePlanExternalName": "standard-128",
+				"clusterServicePlanRef": {
+					"name": "8d659058-a3b4-4058-b039-cc03a31b9442"
+				},
+				"externalID": "f5d24838-b765-4988-8381-3f2d0945ddc1",
+				"updateRequests": 0,
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:catalog",
+						"system:authenticated"
+					],
+					"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+					"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": false,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:44:38Z",
+						"message": "The instance was provisioned successfully",
+						"reason": "ProvisionedSuccessfully",
+						"status": "True",
+						"type": "Ready"
+					}
+				],
+				"deprovisionStatus": "Required",
+				"externalProperties": {
+					"clusterServicePlanExternalID": "8d659058-a3b4-4058-b039-cc03a31b9442",
+					"clusterServicePlanExternalName": "standard-128",
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:catalog",
+							"system:authenticated"
+						],
+						"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+						"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+					}
+				},
+				"lastConditionState": "Ready",
+				"observedGeneration": 2,
+				"orphanMitigationInProgress": false,
+				"provisionStatus": "Provisioned",
+				"reconciledGeneration": 2,
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
+				"userSpecifiedPlanName": "standard-128"
+			}
+		},
+		"oldObject": {
+			"apiVersion": "servicecatalog.k8s.io/v1beta1",
+			"kind": "ServiceInstance",
+			"metadata": {
+				"creationTimestamp": "2024-03-25T01:34:27Z",
+				"finalizers": [
+					"kubernetes-incubator/service-catalog"
+				],
+				"generation": 2,
+				"labels": {
+					"heritage": "drycc",
+					"servicecatalog.k8s.io/spec.clusterServiceClassRef.name": "d1225ad72f9ea4317aac1415103d81e947e19a27c681bd02193ed278",
+					"servicecatalog.k8s.io/spec.clusterServicePlanRef.name": "c243d1d2925e51b1116b0b5d9a0d240d8847a97b2819405b483fc650"
+				},
+				"managedFields": [
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:metadata": {
+								"f:labels": {
+									".": {},
+									"f:heritage": {}
+								}
+							},
+							"f:spec": {
+								".": {},
+								"f:clusterServiceClassExternalName": {},
+								"f:clusterServicePlanExternalName": {}
+							}
+						},
+						"manager": "Drycc Controller",
+						"operation": "Update",
+						"time": "2024-03-25T01:34:27Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:spec": {
+								"f:clusterServiceClassRef": {
+									".": {},
+									"f:name": {}
+								},
+								"f:clusterServicePlanRef": {
+									".": {},
+									"f:name": {}
+								}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"time": "2024-03-25T01:42:28Z"
+					},
+					{
+						"apiVersion": "servicecatalog.k8s.io/v1beta1",
+						"fieldsType": "FieldsV1",
+						"fieldsV1": {
+							"f:status": {
+								".": {},
+								"f:asyncOpInProgress": {},
+								"f:conditions": {},
+								"f:currentOperation": {},
+								"f:deprovisionStatus": {},
+								"f:inProgressProperties": {
+									".": {},
+									"f:clusterServicePlanExternalID": {},
+									"f:clusterServicePlanExternalName": {},
+									"f:userInfo": {
+										".": {},
+										"f:extra": {
+											".": {},
+											"f:authentication.kubernetes.io/pod-name": {},
+											"f:authentication.kubernetes.io/pod-uid": {}
+										},
+										"f:groups": {},
+										"f:uid": {},
+										"f:username": {}
+									}
+								},
+								"f:lastConditionState": {},
+								"f:observedGeneration": {},
+								"f:operationStartTime": {},
+								"f:orphanMitigationInProgress": {},
+								"f:provisionStatus": {},
+								"f:reconciledGeneration": {},
+								"f:userSpecifiedClassName": {},
+								"f:userSpecifiedPlanName": {}
+							}
+						},
+						"manager": "service-catalog",
+						"operation": "Update",
+						"subresource": "status",
+						"time": "2024-03-25T01:42:31Z"
+					}
+				],
+				"name": "valkey-t2",
+				"namespace": "myapp",
+				"resourceVersion": "74040375",
+				"uid": "0c9c3c73-d1b7-4183-9eb5-77ce93f8f8d8"
+			},
+			"spec": {
+				"clusterServiceClassExternalName": "valkey",
+				"clusterServiceClassRef": {
+					"name": "e1fd0d37-9046-4152-a29b-d155c5657c8b"
+				},
+				"clusterServicePlanExternalName": "standard-128",
+				"clusterServicePlanRef": {
+					"name": "8d659058-a3b4-4058-b039-cc03a31b9442"
+				},
+				"externalID": "f5d24838-b765-4988-8381-3f2d0945ddc1",
+				"updateRequests": 0,
+				"userInfo": {
+					"extra": {
+						"authentication.kubernetes.io/pod-name": [
+							"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+						],
+						"authentication.kubernetes.io/pod-uid": [
+							"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+						]
+					},
+					"groups": [
+						"system:serviceaccounts",
+						"system:serviceaccounts:catalog",
+						"system:authenticated"
+					],
+					"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+					"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+				}
+			},
+			"status": {
+				"asyncOpInProgress": true,
+				"conditions": [
+					{
+						"lastTransitionTime": "2024-03-25T01:42:28Z",
+						"message": "The instance is being provisioned asynchronously (provision f5d24838-b765-4988-8381-3f2d0945ddc1 in progress at 1711330950.8151164)",
+						"reason": "Provisioning",
+						"status": "False",
+						"type": "Ready"
+					}
+				],
+				"currentOperation": "Provision",
+				"deprovisionStatus": "Required",
+				"inProgressProperties": {
+					"clusterServicePlanExternalID": "8d659058-a3b4-4058-b039-cc03a31b9442",
+					"clusterServicePlanExternalName": "standard-128",
+					"userInfo": {
+						"extra": {
+							"authentication.kubernetes.io/pod-name": [
+								"catalog-catalog-controller-manager-f7f8c8c4-v5pbs"
+							],
+							"authentication.kubernetes.io/pod-uid": [
+								"dd51f5be-c83f-452e-99c1-475275dbcbc8"
+							]
+						},
+						"groups": [
+							"system:serviceaccounts",
+							"system:serviceaccounts:catalog",
+							"system:authenticated"
+						],
+						"uid": "b06a6359-7a1e-4638-9eb0-c93dfeff7171",
+						"username": "system:serviceaccount:catalog:service-catalog-controller-manager"
+					}
+				},
+				"lastConditionState": "Provisioning",
+				"observedGeneration": 2,
+				"operationStartTime": "2024-03-25T01:42:28Z",
+				"orphanMitigationInProgress": false,
+				"provisionStatus": "",
+				"reconciledGeneration": 0,
+				"userSpecifiedClassName": "ClusterServiceClass/valkey",
+				"userSpecifiedPlanName": "standard-128"
+			}
+		},
+		"dryRun": false,
+		"options": {
+			"kind": "UpdateOptions",
+			"apiVersion": "meta.k8s.io/v1"
+		}
+	}
+}
\ No newline at end of file
diff --git a/rootfs/api/tests/rules/grpc_rules.json b/rootfs/api/tests/rules/grpc_rules.json
index 925787558..6a882a0a3 100644
--- a/rootfs/api/tests/rules/grpc_rules.json
+++ b/rootfs/api/tests/rules/grpc_rules.json
@@ -1,56 +1,33 @@
-{
-	"stable": [{
-			"matches": [{
+[
+	{
+		"matches": [
+			{
 				"method": {
 					"service": "com.example.User",
 					"method": "Login"
 				}
-			}],
-			"backendRefs": [{
+			}
+		],
+		"backendRefs": [
+			{
 				"name": "foo-svc",
 				"port": 50051
-			}]
-		},
-		{
-			"matches": [{
+			}
+		]
+	},
+	{
+		"matches": [
+			{
 				"method": {
 					"service": "grpc.reflection.v1.ServerReflection"
 				}
-			}],
-			"backendRefs": [{
+			}
+		],
+		"backendRefs": [
+			{
 				"name": "foo-svc",
 				"port": 50051
-			}]
-		}
-	],
-	"canary": [{
-			"matches": [{
-				"method": {
-					"service": "com.example.User",
-					"method": "Login"
-				}
-			}],
-			"backendRefs": [{
-				"name": "foo-svc",
-				"port": 50051
-			}, {
-				"name": "foo-svc-canary",
-				"port": 50051
-			}]
-		},
-		{
-			"matches": [{
-				"method": {
-					"service": "grpc.reflection.v1.ServerReflection"
-				}
-			}],
-			"backendRefs": [{
-				"name": "foo-svc",
-				"port": 50051
-			}, {
-				"name": "foo-svc-canary",
-				"port": 50051
-			}]
-		}
-	]
-}
\ No newline at end of file
+			}
+		]
+	}
+]
\ No newline at end of file
diff --git a/rootfs/api/tests/rules/http_rules.json b/rootfs/api/tests/rules/http_rules.json
index 4021ea883..a7cf7b6b0 100644
--- a/rootfs/api/tests/rules/http_rules.json
+++ b/rootfs/api/tests/rules/http_rules.json
@@ -1,69 +1,43 @@
-{
-	"stable": [{
-		"matches": [{
-			"path": {
-				"type": "PathPrefix",
-				"value": "/add-multiple-response-headers"
-			}
-		}],
-		"filters": [{
-			"type": "ResponseHeaderModifier",
-			"responseHeaderModifier": {
-				"add": [{
-						"name": "X-Header-Add-1",
-						"value": "header-add-1"
-					},
-					{
-						"name": "X-Header-Add-2",
-						"value": "header-add-2"
-					},
-					{
-						"name": "X-Header-Add-3",
-						"value": "header-add-3"
-					}
-				]
-			}
-		}],
-		"backendRefs": [{
-			"name": "echo",
-			"port": 8080
-		}]
-	}],
-	"canary": [{
-		"matches": [{
-			"path": {
-				"type": "PathPrefix",
-				"value": "/add-multiple-response-headers"
+[
+	{
+		"matches": [
+			{
+				"path": {
+					"type": "PathPrefix",
+					"value": "/add-multiple-response-headers"
+				}
 			}
-		}],
-		"filters": [{
-			"type": "ResponseHeaderModifier",
-			"responseHeaderModifier": {
-				"add": [{
-						"name": "X-Header-Add-1",
-						"value": "header-add-1"
-					},
-					{
-						"name": "X-Header-Add-2",
-						"value": "header-add-2"
-					},
-					{
-						"name": "X-Header-Add-3",
-						"value": "header-add-3"
-					}
-				]
+		],
+		"filters": [
+			{
+				"type": "ResponseHeaderModifier",
+				"responseHeaderModifier": {
+					"add": [
+						{
+							"name": "X-Header-Add-1",
+							"value": "header-add-1"
+						},
+						{
+							"name": "X-Header-Add-2",
+							"value": "header-add-2"
+						},
+						{
+							"name": "X-Header-Add-3",
+							"value": "header-add-3"
+						}
+					]
+				}
 			}
-		}],
-		"backendRefs": [{
-				"name": "echo",
-				"port": 8080,
-				"weight": 10
-			},
+		],
+		"backendRefs": [
 			{
-				"name": "echo-canary",
-				"port": 8080,
-				"weight": 90
+				"name": "echo",
+				"port": 8080
 			}
-		]
-	}]
-}
\ No newline at end of file
+		],
+		"timeouts": {
+			"backendRequest": "60s",
+			"request": "60s"
+		}
+	}
+]
\ No newline at end of file
diff --git a/rootfs/api/tests/rules/tcp_rules.json b/rootfs/api/tests/rules/tcp_rules.json
index d13cab8cc..0ca5ea93f 100644
--- a/rootfs/api/tests/rules/tcp_rules.json
+++ b/rootfs/api/tests/rules/tcp_rules.json
@@ -1,21 +1,11 @@
-{
-	"stable": [{
-		"backendRefs": [{
-			"name": "echo",
-			"port": 8080
-		}]
-	}],
-	"canary": [{
-		"backendRefs": [{
+[
+	{
+		"backendRefs": [
+			{
 				"name": "echo",
 				"port": 8080,
 				"weight": 10
-			},
-			{
-				"name": "echo-canary",
-				"port": 8080,
-				"weight": 90
 			}
 		]
-	}]
-}
\ No newline at end of file
+	}
+]
\ No newline at end of file
diff --git a/rootfs/api/tests/test_admissions.py b/rootfs/api/tests/test_admissions.py
new file mode 100644
index 000000000..00b9463ec
--- /dev/null
+++ b/rootfs/api/tests/test_admissions.py
@@ -0,0 +1,83 @@
+import os
+import json
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from api.models.app import App
+from api.models.resource import Resource
+from api import admissions
+from api.tests import adapter, TEST_ROOT, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+SCALE_TEST_CASES = (
+    ("deployment_scale.json", (admissions.DeploymentsScaleHandler, "web", 2)),
+    ("job_status_create_ok.json", (admissions.JobsStatusHandler, "run", 1)),
+    ("job_status_succeeded.json", (admissions.JobsStatusHandler, "run", 0)),
+    ("job_status_create_ok.json", (admissions.JobsStatusHandler, "run", 1)),
+    ("job_status_failed.json", (admissions.JobsStatusHandler, "run", 0)),
+)
+
+RESOURCE_TEST_CASES = (
+    (
+        "service_instance_provisioning.json",
+        (admissions.ServiceInstancesStatusHandler, "Provisioning", None),
+    ),
+    (
+        "service_instance_ready.json",
+        (admissions.ServiceInstancesStatusHandler, "Ready", None),
+    ),
+    (
+        "service_binding_binding_request_in_flight.json",
+        (admissions.ServicebindingsStatusHandler, "Ready", "BindingRequestInFlight"),
+    ),
+    (
+        "service_binding_ready.json",
+        (admissions.ServicebindingsStatusHandler, "Ready", "Ready"),
+    ),
+)
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class AdmissionsTest(DryccTransactionTestCase):
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+    def tearDown(self):
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def test_admissions_scale(self, requests_mock):
+        app_id = self.create_app("myapp")
+        for case, (admission_class, ptype, scale) in SCALE_TEST_CASES:
+            with open(os.path.join(TEST_ROOT, "admissions", case)) as f:
+                request = json.loads(f.read())["request"]
+                handler = admission_class()
+                self.assertEqual(handler.detect(request), True)
+                self.assertEqual(handler.handle(request), True)
+                app = App.objects.get(id=app_id)
+                self.assertEqual(app.structure.get(ptype, 0), scale)
+
+    def test_admissions_service_catalog(self, requests_mock):
+        app_id = self.create_app("myapp")
+        resource_name = 'valkey-t2'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': resource_name, 'plan': 'valkey:standard-128'}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+
+        for case, (admission_class, status, ready) in RESOURCE_TEST_CASES:
+            with open(os.path.join(TEST_ROOT, "admissions", case)) as f:
+                request = json.loads(f.read())["request"]
+                handler = admission_class()
+                data = (handler, f)
+                self.assertEqual(handler.detect(request), True, data)
+                self.assertEqual(handler.handle(request), True, data)
+                resource = Resource.objects.get(name=resource_name)
+                self.assertEqual(resource.status, status, data)
+                self.assertEqual(resource.binding, ready, data)
diff --git a/rootfs/api/tests/test_api_middleware.py b/rootfs/api/tests/test_api_middleware.py
index 460111e49..1fd270de6 100644
--- a/rootfs/api/tests/test_api_middleware.py
+++ b/rootfs/api/tests/test_api_middleware.py
@@ -5,7 +5,6 @@
 """
 
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 from api.tests import DryccTestCase
 
 from api import __version__
@@ -21,7 +20,7 @@ class APIMiddlewareTest(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def test_drycc_version_header_good(self):
diff --git a/rootfs/api/tests/test_app.py b/rootfs/api/tests/test_app.py
index 7015e16d5..162c8ae19 100644
--- a/rootfs/api/tests/test_app.py
+++ b/rootfs/api/tests/test_app.py
@@ -14,10 +14,10 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 from django.test.utils import override_settings
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
-from api.models.config import Config
+from api.models.base import PTYPE_WEB
+from api.models.workspace import Workspace, WorkspaceMember
 from scheduler import KubeException, KubeHTTPException
 
 from api.exceptions import DryccException
@@ -43,8 +43,20 @@ class AppTest(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.workspace_name = self._ensure_workspace_admin(self._default_workspace_name())
+
+        original_post = self.client.post
+
+        def _post_with_workspace(path, data=None, *args, **kwargs):
+            if path == '/v2/apps':
+                payload = {} if data is None else dict(data)
+                payload.setdefault('workspace', self.workspace_name)
+                data = payload
+            return original_post(path, data, *args, **kwargs)
+
+        self.client.post = _post_with_workspace
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
@@ -60,13 +72,14 @@ def test_app(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 1)
 
-        url = '/v2/apps/{app_id}'.format(**locals())
+        url = f'/v2/apps/{app_id}'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
         body = {'id': 'new'}
         response = self.client.patch(url, body)
-        self.assertEqual(response.status_code, 405, response.content)
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(str(response.data['detail']), 'workspace is required')
 
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
@@ -90,14 +103,13 @@ def test_response_data(self, mock_requests):
         body = {'id': 'app-{}'.format(random.randrange(1000, 10000))}
         response = self.client.post('/v2/apps', body)
         for key in response.data:
-            self.assertIn(key, ['uuid', 'created', 'updated', 'id', 'owner', 'structure',
-                                'procfile_structure'])
+            self.assertIn(key, ['uuid', 'created', 'updated', 'id', 'workspace', 'structure'])
         expected = {
             'id': body['id'],
-            'owner': self.user.username,
+            'workspace': self.user.username,
             'structure': {}
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, expected | response.data)
 
     def test_app_override_id(self, mock_requests):
         app_id = self.create_app()
@@ -108,19 +120,20 @@ def test_app_override_id(self, mock_requests):
     @mock.patch('api.models.app.logger')
     def test_app_release_notes_in_logs(self, mock_requests, mock_logger):
         """Verifies that an app's release summary is dumped into the logs."""
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
-
-        # check app logs
-        exp_msg = "[{app_id}]: {self.user.username} created initial release".format(**locals())
-        mock_logger.log.has_calls(logging.INFO, exp_msg)
-        app.log('hello world')
-        exp_msg = "[{app_id}]: hello world".format(**locals())
-        mock_logger.log.has_calls(logging.INFO, exp_msg)
-        app.log('goodbye world', logging.WARNING)
-        # assert logging with a different log level
-        exp_msg = "[{app_id}]: goodbye world".format(**locals())
-        mock_logger.log.has_calls(logging.WARNING, exp_msg)
+        with mock.patch('api.models.release.logger') as release_logger:
+            app_id = self.create_app()
+            app = App.objects.get(id=app_id)
+            # check release logs
+            exp_msg = "[{app_id}]: {self.user.username} created initial release".format(
+                **locals())
+            release_logger.log.assert_any_call(logging.INFO, exp_msg)
+            app.log('hello world')
+            exp_msg = f"[{app_id}]: hello world"
+            mock_logger.log.assert_any_call(logging.INFO, exp_msg)
+            app.log('goodbye world', logging.WARNING)
+            # assert logging with a different log level
+            exp_msg = f"[{app_id}]: goodbye world"
+            mock_logger.log.assert_any_call(logging.WARNING, exp_msg)
 
     def test_app_errors(self, mock_requests):
         response = self.client.post('/v2/apps', {'id': 'camelCase'})
@@ -148,18 +161,18 @@ def test_app_errors(self, mock_requests):
         )
 
         app_id = self.create_app()
-        url = '/v2/apps/{app_id}'.format(**locals())
+        url = f'/v2/apps/{app_id}'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
         for endpoint in ('containers', 'config', 'releases', 'builds'):
-            url = '/v2/apps/{app_id}/{endpoint}'.format(**locals())
+            url = f'/v2/apps/{app_id}/{endpoint}'
             response = self.client.get(url)
             self.assertEqual(response.status_code, 404)
 
     def test_app_reserved_names(self, mock_requests):
         """Nobody should be able to create applications with names which are reserved."""
-        reserved_names = ['foo', 'bar']
-        with self.settings(DRYCC_RESERVED_NAMES=reserved_names):
+        reserved_names = ['fooooo', 'barrrrrr']
+        with self.settings(RESERVED_NAME_PATTERNS=reserved_names):
             for name in reserved_names:
                 response = self.client.post('/v2/apps', {'id': name})
                 self.assertContains(
@@ -183,26 +196,32 @@ def test_app_structure_is_valid_json(self, mock_requests):
         self.assertIn('structure', response.data)
         self.assertEqual(response.data['structure'], {"web": 1})
 
-    @mock.patch('api.models.app.logger')
+    @mock.patch('api.models.release.logger')
     def test_admin_can_manage_other_apps(self, mock_requests, mock_logger):
         """Administrators of Drycc should be able to manage all applications.
         """
         # log in as non-admin user and create an app
-        user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        username = 'autotest2'
+        user = User.objects.get(username=username)
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
         app_id = self.create_app()
 
+        app = App.objects.get(id=app_id)
+        WorkspaceMember.objects.get_or_create(
+            workspace=app.workspace,
+            user=self.user,
+            defaults={'role': 'admin'},
+        )
+
         # log in as admin, check to see if they have access
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         url = '/v2/apps/{}'.format(app_id)
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         # check app logs
-        exp_msg = "autotest2 created initial release"
-        exp_log_call = mock.call(logging.INFO, exp_msg)
-        mock_logger.log.has_calls(exp_log_call)
-
+        exp_msg = "[%s]: %s created initial release" % (app_id, username)
+        mock_logger.log.assert_any_call(logging.INFO, exp_msg)
         # TODO: test run needs an initial build
         # delete the app
         url = '/v2/apps/{}'.format(app_id)
@@ -215,9 +234,16 @@ def test_admin_can_see_other_apps(self, mock_requests):
         """
         # log in as non-admin user and create an app
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
-        self.create_app()
+        app_id = self.create_app()
+
+        app = App.objects.get(id=app_id)
+        WorkspaceMember.objects.get_or_create(
+            workspace=app.workspace,
+            user=self.user,
+            defaults={'role': 'viewer'},
+        )
 
         # log in as admin
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
@@ -235,8 +261,8 @@ def test_run_without_release_should_error(self, mock_requests):
         body = {'command': 'ls -al'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {'detail': 'No build associated with this '
-                                                   'release to run this command'})
+        self.assertEqual(
+            str(response.data["detail"]), 'no build available, please deploy a release')
 
     @mock.patch('api.models.app.App.run', _mock_run)
     @mock.patch('api.models.app.App.deploy', mock_none)
@@ -248,7 +274,7 @@ def test_run(self, mock_requests):
 
         # create build
         body = {'image': 'autotest/example', 'stack': 'container'}
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -256,14 +282,15 @@ def test_run(self, mock_requests):
         url = '/v2/apps/{}/run'.format(app_id)
         response = self.client.post(url, {})
         self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {'detail': 'command is a required field'})
+        self.assertEqual(
+            response.data,
+            {'detail': 'command is a required field, or it can be defined in Procfile'}
+        )
 
         # run command
         body = {'command': 'ls -al'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['exit_code'], 0)
-        self.assertEqual(response.data['output'], 'mock')
+        self.assertEqual(response.status_code, 204, response.data)
 
     def test_run_failure(self, mock_requests):
         """Raise a KubeException via scheduler.run"""
@@ -271,11 +298,11 @@ def test_run_failure(self, mock_requests):
 
         # create build
         body = {'image': 'autotest/example', 'stack': 'container'}
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        with mock.patch('scheduler.KubeHTTPClient.run') as kube_run:
+        with mock.patch('scheduler.KubeHTTPClient.http_post') as kube_run:
             kube_run.side_effect = KubeException('boom!')
             # run command
             url = '/v2/apps/{}/run'.format(app_id)
@@ -288,24 +315,24 @@ def test_unauthorized_user_cannot_see_app(self, mock_requests):
         An unauthorized user should not be able to access an app's resources.
 
         Since an unauthorized user can't access the application, these
-        tests should return a 403, but currently return a 404. FIXME!
+        tests return 404 when app is filtered out from queryset.
         """
         app_id = self.create_app()
         unauthorized_user = User.objects.get(username='autotest2')
-        unauthorized_token = Token.objects.get(user=unauthorized_user).key
+        unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
 
         url = '/v2/apps/{}/run'.format(app_id)
         body = {'command': 'foo'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
         url = '/v2/apps/{}'.format(app_id)
         response = self.client.get(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
         response = self.client.delete(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404)
 
     def test_app_info_not_showing_wrong_app(self, mock_requests):
         self.create_app()
@@ -314,68 +341,82 @@ def test_app_info_not_showing_wrong_app(self, mock_requests):
 
     def test_app_transfer(self, mock_requests):
         owner = User.objects.get(username='autotest2')
-        owner_token = Token.objects.get(user=owner).key
+        owner_token = self.get_or_create_token(owner)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
 
         collaborator = User.objects.get(username='autotest3')
+        collab_token = self.get_or_create_token(collaborator)
 
-        app = App.objects.create(owner=owner)
+        # create a workspace and app under owner
+        response = self.client.post('/v2/workspaces', {
+            'name': 'wstransfer1',
+            'email': 'ws-owner@example.com',
+        })
+        self.assertEqual(response.status_code, 201, response.data)
 
-        # pretend the owner and a collaborator added some config to the app to ensure
-        # resources owned by the owner are transferred, but not resources owned by the
-        # collaborator.
-        config1 = Config.objects.create(owner=owner, app=app, values={'FOO': 'bar'})
-        config2 = Config.objects.create(owner=collaborator, app=app, values={'CAR': 'star'})
+        response = self.client.post('/v2/apps', {
+            'id': 'app-transfer01',
+            'workspace': 'wstransfer1',
+        })
+        self.assertEqual(response.status_code, 201, response.data)
 
-        # Transfer App
+        app = App.objects.get(id='app-transfer01')
+        workspace = Workspace.objects.get(name='wstransfer1')
         url = '/v2/apps/{}'.format(app.id)
-        new_owner = User.objects.get(username='autotest4')
-        new_owner_token = Token.objects.get(user=new_owner).key
-        body = {'owner': new_owner.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
 
-        # Original user can no longer access it
+        # collaborator cannot access app before joining workspace
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
         response = self.client.get(url)
-        self.assertEqual(response.status_code, 403)
+        self.assertEqual(response.status_code, 404, response.data)
+
+        # owner adds collaborator to workspace
+        WorkspaceMember.objects.get_or_create(
+            workspace=workspace,
+            user=collaborator,
+            defaults={'role': 'member'},
+        )
 
-        # New owner can access it
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + new_owner_token)
+        # collaborator can access app after membership
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['owner'], new_owner.username)
-
-        # At this point config1.owner field is still the old owner, but the value in the database
-        # was updated to the new owner when we performed the transfer. The object's updated values
-        # needs to be reloaded from the database to get an accurate idea who owns the object.
-        # https://docs.djangoproject.com/en/dev/ref/models/instances/#django.db.models.Model.refresh_from_db
-        config1.refresh_from_db()
-        config2.refresh_from_db()
-
-        # New owner also is given ownership to all resources owned by the original user, but not
-        # resources created by other users
-        self.assertEqual(config1.owner, new_owner)
-        self.assertEqual(config2.owner, collaborator)
-
-        # Collaborators can't transfer
-        body = {'username': owner.username}
-        perms_url = url+"/perms/"
-        response = self.client.post(perms_url, body)
+
+        # collaborator cannot manage workspace members
+        response = self.client.patch(
+            f'/v2/workspaces/{workspace.name}/members/{owner.username}',
+            {'role': 'viewer'},
+        )
+        self.assertEqual(response.status_code, 403, response.data)
+
+    def test_transfer_api_success(self, mock_requests):
+        app_id = self.create_app(name='appxferok')
+        response = self.client.post('/v2/workspaces', {
+            'name': 'wstarget',
+            'email': 'target@example.com',
+        })
         self.assertEqual(response.status_code, 201, response.data)
 
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
-        body = {'owner': self.user.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'wstarget'},
+        )
+        self.assertEqual(response.status_code, 204, response.data)
 
-        # Admins can transfer
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {'owner': self.user.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['owner'], self.user.username)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.workspace.name, 'wstarget')
+
+    def test_transfer_api_requires_workspace(self, mock_requests):
+        app_id = self.create_app(name='appxferreqws')
+        response = self.client.patch(f'/v2/apps/{app_id}', {})
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(str(response.data['detail']), 'workspace is required')
+
+    def test_transfer_api_target_workspace_not_found(self, mock_requests):
+        app_id = self.create_app(name='appxfer404')
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'workspace-not-exists'},
+        )
+        self.assertEqual(response.status_code, 404, response.data)
 
     def test_app_exists_in_kubernetes(self, mock_requests):
         """
@@ -446,12 +487,11 @@ def test_app_verify_application_health_success(self, mock_requests):
 
         # create app
         app_id = self.create_app()
-        hostname = 'http://{}.{}.svc.{}:80/'.format(
-            app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
+        hostname = 'http://{}.{}.svc:80/'.format(app_id, app_id)
         mr = mock_requests.register_uri('GET', hostname, responses)
 
         # deploy app to get verification
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -482,11 +522,10 @@ def test_app_verify_application_health_failure_404(self, mock_requests):
         # create app
         app_id = self.create_app()
 
-        hostname = 'http://{}.{}.svc.{}:80/'.format(
-            app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
+        hostname = 'http://{}.{}.svc:80/'.format(app_id, app_id)
         mr = mock_requests.register_uri('GET', hostname, responses)
         # deploy app to get verification
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -506,12 +545,11 @@ def _raise_exception(request, ctx):
         # create app
         app_id = self.create_app()
         # function tries to hit router 10 times
-        hostname = 'http://{}.{}.svc.{}:80/'.format(
-            app_id, app_id, settings.KUBERNETES_CLUSTER_DOMAIN)
+        hostname = 'http://{}.{}.svc:80/'.format(app_id, app_id)
         mr = mock_requests.register_uri('GET', hostname, text=_raise_exception)
 
         # deploy app to get verification
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -525,7 +563,7 @@ def test_list_ordering(self, mock_requests):
         """
         Test that a list of apps is sorted by name
         """
-        for name in ['zulu', 'tango', 'alpha', 'foxtrot']:
+        for name in ['zulua', 'tango', 'alpha', 'foxtrot']:
             response = self.client.post('/v2/apps', {'id': name})
             self.assertEqual(response.status_code, 201, response.data)
 
@@ -534,27 +572,27 @@ def test_list_ordering(self, mock_requests):
         self.assertEqual(apps[0]['id'], 'alpha')
         self.assertEqual(apps[1]['id'], 'foxtrot')
         self.assertEqual(apps[2]['id'], 'tango')
-        self.assertEqual(apps[3]['id'], 'zulu')
+        self.assertEqual(apps[3]['id'], 'zulua')
 
     def test_get_private_registry_config(self, mock_requests):
-        registry = {'username': 'test', 'password': 'test'}
+        registry = {"web": {'username': 'test', 'password': 'test'}}
         auth = bytes('{}:{}'.format("test", "test"), 'UTF-8')
         encAuth = base64.b64encode(auth).decode(encoding='UTF-8')
         image = 'test/test'
 
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry.get("web", {}))  # noqa
         dockerConfig = json.loads(docker_config)
         expected = {"https://index.docker.io/v1/": {"auth": encAuth}}
         self.assertEqual(dockerConfig.get('auths'), expected)
-        self.assertEqual(name, "private-registry")
+        self.assertEqual(name, "private-registry-web")
         self.assertEqual(create, True)
 
         image = "quay.io/test/test"
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry.get("web", {}))  # noqa
         dockerConfig = json.loads(docker_config)
         expected = {"quay.io": {"auth": encAuth}}
         self.assertEqual(dockerConfig.get('auths'), expected)
-        self.assertEqual(name, "private-registry")
+        self.assertEqual(name, "private-registry-web")
         self.assertEqual(create, True)
 
     @override_settings(REGISTRY_LOCATION="off-cluster")
@@ -563,36 +601,36 @@ def test_get_private_registry_config_off_cluster(self, mock_requests):
         auth = bytes('{}:{}'.format("test", "test"), 'UTF-8')
         encAuth = base64.b64encode(auth).decode(encoding='UTF-8')
         image = "test.com/test/test"
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry.get("web", {}))  # noqa
         dockerConfig = json.loads(docker_config)
         expected = {"https://index.docker.io/v1/": {
             "auth": encAuth
         }}
         self.assertEqual(dockerConfig.get('auths'), expected)
-        self.assertEqual(name, "private-registry-off-cluster")
+        self.assertEqual(name, "private-registry-web-off-cluster")
         self.assertEqual(create, True)
 
     @override_settings(REGISTRY_LOCATION="ecra")
     def test_get_private_registry_config_bad_registry_location(self, mock_requests):
         registry = {}
         image = "test.com/test/test"
-        docker_config, name, create = App()._get_private_registry_config(image, registry)
+        docker_config, name, create = App()._get_private_registry_config("web", image, registry)
         self.assertEqual(docker_config, None)
         self.assertEqual(name, None)
         self.assertEqual(create, None)
 
     def test_build_env_vars(self, mock_requests):
-        app = App.objects.create(owner=self.user)
+        app = App.objects.create(workspace=Workspace.objects.get(name=self.workspace_name))
         # Make sure an exception is raised when calling without a build available
         with self.assertRaises(DryccException):
-            app._build_env_vars(app.release_set.latest())
+            app._build_env_vars(app.release_set.latest(), PTYPE_WEB)
         data = {'image': 'autotest/example', 'stack': 'heroku-18'}
-        url = "/v2/apps/{app.id}/builds".format(**locals())
+        url = f"/v2/apps/{app.id}/build"
         response = self.client.post(url, data)
         self.assertEqual(response.status_code, 201, response.data)
         time_created = app.release_set.latest().created
         self.assertEqual(
-            app._build_env_vars(app.release_set.latest()),
+            app._build_env_vars(app.release_set.latest(), PTYPE_WEB),
             {
                 'DRYCC_APP': app.id,
                 'WORKFLOW_RELEASE': 'v2',
@@ -607,7 +645,7 @@ def test_build_env_vars(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         time_created = app.release_set.latest().created
         self.assertEqual(
-            app._build_env_vars(app.release_set.latest()),
+            app._build_env_vars(app.release_set.latest(), PTYPE_WEB),
             {
                 'DRYCC_APP': app.id,
                 'WORKFLOW_RELEASE': 'v3',
@@ -619,27 +657,34 @@ def test_build_env_vars(self, mock_requests):
             })
 
     def test_gather_app_settings(self, mock_requests):
-        app = App.objects.create(owner=self.user)
+        app = App.objects.create(workspace=Workspace.objects.get(name=self.workspace_name))
         app.save()
         data = {'image': 'autotest/example', 'stack': 'container'}
-        url = "/v2/apps/{app.id}/builds".format(**locals())
+        url = f"/v2/apps/{app.id}/build"
         response = self.client.post(url, data)
         self.assertEqual(response.status_code, 201, response.data)
-        Config.objects.create(
-            owner=self.user,
-            app=app,
-            values={
-                'DRYCC_DEPLOY_TIMEOUT': '60',
-                'DRYCC_DEPLOY_BATCHES': '3',
-                'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS': '60'
-            })
+        # Set some app settings
+        url = f"/v2/apps/{app.id}/config"
+        body = {
+            'values': [
+                {"name": "DRYCC_DEPLOY_TIMEOUT", "value": "60", "group": "global"},
+                {"name": "DRYCC_DEPLOY_BATCHES", "value": "3", "group": "global"},
+                {
+                    "name": "KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS",
+                    "value": "90", "group": "global"
+                },
+            ],
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        # Gather app settings
         s = app._gather_app_settings(app.release_set.latest(),
                                      app.appsettings_set.latest(),
                                      'web',
                                      3)
-        assert isinstance(s['deploy_batches'], int)
-        assert isinstance(s['deploy_timeout'], int)
-        assert isinstance(s['pod_termination_grace_period_seconds'], int)
+        self.assertEqual(s['deploy_batches'], 3)
+        self.assertEqual(s['deploy_timeout'], 60)
+        self.assertEqual(s['termination_grace_period_seconds'], 90)
 
     def test_app_name_bad_regex(self, mock_requests):
         """
@@ -657,7 +702,7 @@ def test_app_name_bad_regex(self, mock_requests):
         self.assertEqual(response.data['results'][0]['id'], app_id, response.data)
 
         # deploy to an app that doesn't exist should fail with 404
-        url = "/v2/apps/{}/builds".format('={}'.format(app_id))
+        url = "/v2/apps/{}/build".format('={}'.format(app_id))
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 404, response)
diff --git a/rootfs/api/tests/test_app_settings.py b/rootfs/api/tests/test_app_settings.py
index 3407fcfb3..794d77004 100644
--- a/rootfs/api/tests/test_app_settings.py
+++ b/rootfs/api/tests/test_app_settings.py
@@ -2,7 +2,6 @@
 
 from django.core.cache import cache
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
 from api.tests import adapter, DryccTransactionTestCase
@@ -18,14 +17,14 @@ class TestAppSettings(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
-    def test_settings_routable(self, mock_requests):
+    def test_settings_bool(self, mock_requests):
         """
         Create an application with the routable flag turned on or off
         """
@@ -33,13 +32,67 @@ def test_settings_routable(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
         self.assertTrue(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
         # Set routable to false
         response = self.client.post(
-            '/v2/apps/{app.id}/settings'.format(**locals()),
+            f'/v2/apps/{app.id}/settings',
             {'routable': False}
         )
         self.assertEqual(response.status_code, 201, response.data)
         self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
+        # Set autodeploy to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autodeploy': False}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
+        # Set autorollback to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autorollback': False}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
+        self.assertFalse(app.appsettings_set.latest().autorollback)
+
+        # Set autorollback to true
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autorollback': True}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertFalse(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
+        # Set autodeploy to true
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'autodeploy': True}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertFalse(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
+
+        # Set routable to false
+        response = self.client.post(
+            f'/v2/apps/{app.id}/settings',
+            {'routable': True}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertTrue(app.appsettings_set.latest().routable)
+        self.assertTrue(app.appsettings_set.latest().autodeploy)
+        self.assertTrue(app.appsettings_set.latest().autorollback)
 
     def test_autoscale(self, mock_requests):
         """
@@ -50,7 +103,7 @@ def test_autoscale(self, mock_requests):
         # create an autoscaling rule
         scale = {'autoscale': {'cmd': {'min': 2, 'max': 5, 'cpu_percent': 45}}}
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             scale
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -60,7 +113,7 @@ def test_autoscale(self, mock_requests):
         # update
         scale = {'autoscale': {'cmd': {'min': 2, 'max': 8, 'cpu_percent': 45}}}
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             scale
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -70,7 +123,7 @@ def test_autoscale(self, mock_requests):
         # create
         scale = {'autoscale': {'worker': {'min': 2, 'max': 5, 'cpu_percent': 45}}}
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             scale
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -82,13 +135,17 @@ def test_autoscale(self, mock_requests):
 
         # check that config fails if trying to unset non-existing proc type
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
-            {'autoscale': {'invalid_proctype': None}})
+            f'/v2/apps/{app_id}/settings',
+            {'autoscale': {'invalid-proctype': None}})
         self.assertEqual(response.status_code, 422, response.data)
+        response = self.client.post(
+            f'/v2/apps/{app_id}/settings',
+            {'autoscale': {'invalid_proctype': None}})
+        self.assertEqual(response.status_code, 400, response.data)
 
         # remove a proc type
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             {'autoscale': {'worker': None}})
         self.assertEqual(response.status_code, 201, response.data)
         self.assertNotIn('worker', response.data['autoscale'])
@@ -96,7 +153,7 @@ def test_autoscale(self, mock_requests):
 
         # remove another proc type
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             {'autoscale': {'cmd': None}})
         self.assertEqual(response.status_code, 201, response.data)
         self.assertNotIn('cmd', response.data['autoscale'])
@@ -109,14 +166,14 @@ def test_autoscale_validations(self, mock_requests):
 
         # Set one of the values that require a numeric value to a string
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             {'autoscale': {'cmd': {'min': 4, 'max': 5, 'cpu_percent': "t"}}}
         )
         self.assertEqual(response.status_code, 400, response.data)
 
         # Don't set one of the mandatory value
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             {'autoscale': {'cmd': {'min': 4, 'cpu_percent': 45}}}
         )
         self.assertEqual(response.status_code, 400, response.data)
@@ -137,7 +194,7 @@ def test_settings_labels(self, mock_requests):
                 }
         }
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             base_labels
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -146,7 +203,7 @@ def test_settings_labels(self, mock_requests):
         # update
         labels = {'label': {'team': 'backend'}}
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             labels
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -157,7 +214,7 @@ def test_settings_labels(self, mock_requests):
         # remove
         labels = {'label': {'git_repo': None}}
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             labels
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -168,45 +225,7 @@ def test_settings_labels(self, mock_requests):
         # error on remove non-exist label
         labels = {'label': {'git_repo': None}}
         response = self.client.post(
-            '/v2/apps/{app_id}/settings'.format(**locals()),
+            f'/v2/apps/{app_id}/settings',
             labels
         )
         self.assertEqual(response.status_code, 422, response.data)
-
-    def test_canaries(self, mock_requests):
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
-        expect = ["web", "task", "new", "apps"]
-        for index, scale_type in enumerate(expect):
-            response = self.client.post(
-                '/v2/apps/{}/services'.format(app_id),
-                {
-                    'port': 5000 + index,
-                    'protocol': 'TCP',
-                    'target_port': 5000 + index,
-                    'procfile_type': scale_type
-                }
-            )
-            self.assertEqual(response.status_code, 201, response.data)
-        self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': expect[:2]}
-        )
-        response = self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': expect[2:]}
-        )
-        expect = ["web", "task", "new", "apps"]
-        self.assertEqual(
-            response.json()["canaries"],
-            expect,
-            response.data
-        )
-
-        self.assertEqual(len(app.service_set.all()), 4)
-        response = self.client.delete(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': ["new", "apps"]}
-        )
-        app_settings = app.appsettings_set.latest()
-        self.assertEqual(app_settings.canaries, ["web", "task"])
diff --git a/rootfs/api/tests/test_build.py b/rootfs/api/tests/test_build.py
index 4b9b2b909..11820bf19 100644
--- a/rootfs/api/tests/test_build.py
+++ b/rootfs/api/tests/test_build.py
@@ -7,12 +7,11 @@
 
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.test.utils import override_settings
 from unittest import mock
-from rest_framework.authtoken.models import Token
 
 from api.models.build import Build
 from api.models.app import App
+from api.models.base import PTYPE_WEB
 from scheduler import KubeException
 
 from api.tests import adapter, DryccTransactionTestCase
@@ -30,7 +29,7 @@ class BuildTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -44,28 +43,26 @@ def test_build(self, mock_requests):
         app_id = self.create_app()
 
         # check to see that no initial build was created
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 0)
 
         # post a new build
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        build_id = str(response.data['uuid'])
         build1 = response.data
         self.assertEqual(response.data['image'], body['image'])
 
         # read the build
-        url = "/v2/apps/{app_id}/builds/{build_id}".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         build2 = response.data
         self.assertEqual(build1, build2)
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -86,15 +83,14 @@ def test_response_data(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
 
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'dockerfile',
-                                'image', 'stack', 'procfile', 'sha'])
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'dockerfile',
+                                'dryccfile', 'image', 'stack', 'procfile', 'sha'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'dockerfile': '',
             'image': 'autotest/example',
@@ -102,31 +98,26 @@ def test_response_data(self, mock_requests):
             'procfile': {},
             'sha': ''
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, expected | response.data)
 
     def test_build_default_containers(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # post an image as a build with a procfile
         app_id = self.create_app()
         # post an image as a build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -137,20 +128,15 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # start with a new app
         app_id = self.create_app()
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -160,21 +146,15 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
-
+        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
         # start with a new app
         app_id = self.create_app()
 
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -187,21 +167,15 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
-
+        self.assertEqual(response.data['results'], [])
         # start with a new app
         app_id = self.create_app()
         # post a new build with procfile
 
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -214,21 +188,16 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # start with a new app
         app_id = self.create_app()
         # post a new build with procfile and no routable type
 
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -241,7 +210,6 @@ def test_build_default_containers(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-    @override_settings(DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE=True)
     def test_build_forgotten_procfile(self, mock_requests):
         """
         Test that when a user first posts a build with a Procfile
@@ -252,7 +220,7 @@ def test_build_forgotten_procfile(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -264,49 +232,54 @@ def test_build_forgotten_procfile(self, mock_requests):
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
+        # verify web
+        url = f"/v2/apps/{app_id}/pods/"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # scale worker
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'worker': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
         # verify worker
-        url = "/v2/apps/{app_id}/pods/worker".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'worker')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-worker-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        # verify worker is not there
-        url = "/v2/apps/{app_id}/pods/worker".format(**locals())
+        # verify worker is not there, only web
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 0)
+        self.assertEqual([item for item in response.data['results'] if item["type"] != "web"], [])
 
-        # verify web is not there
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
+        self.assertEqual(response.json()['structure'], {'web': 1})
+
+        # do another deploy for this time forget Procfile
+        url = f"/v2/apps/{app_id}/build"
+        body = {'image': 'autotest/example:v3', 'stack': 'container'}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
 
         # look at the app structure
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.json()['structure'], {'web': 1, 'worker': 0})
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
-    @override_settings(DRYCC_DEPLOY_PROCFILE_MISSING_REMOVE=False)
     def test_build_no_remove_process(self, mock_requests):
         """
         Specifically test PROCFILE_REMOVE_PROCS_ON_DEPLOY being turned off
@@ -315,142 +288,131 @@ def test_build_no_remove_process(self, mock_requests):
         """
         # start with a new app
         app_id = self.create_app()
+        # Set routable to false
+        response = self.client.post(f'/v2/apps/{app_id}/settings', {'autodeploy': False})
+        self.assertEqual(response.status_code, 201, response.data)
 
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
             'stack': 'heroku-18',
             'procfile': {
                 'web': 'node server.js',
+                'task': 'node task.js',
                 'worker': 'node worker.js'
             }
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
+        # deploy
+        response = self.client.post(f"/v2/apps/{app_id}/releases/deploy/")
+        self.assertEqual(response.status_code, 204, response.data)
+
         # scale worker
-        url = "/v2/apps/{app_id}/scale".format(**locals())
-        body = {'worker': 1}
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'worker': 1, 'task': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
         # verify worker
-        url = "/v2/apps/{app_id}/pods/worker".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'worker')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-worker-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'task', "v2", "up")
+        self.assertPodContains(response.data['results'], app_id, 'worker', "v2", "up")
 
         # do another deploy for this time forget Procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
-        body = {'image': 'autotest/example', 'stack': 'container'}
+        url = f"/v2/apps/{app_id}/build"
+        body = {'image': 'autotest/example:v10', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        # verify worker is still there
-        url = "/v2/apps/{app_id}/pods/worker".format(**locals())
+        # deploy
+        response = self.client.post(f"/v2/apps/{app_id}/releases/deploy/")
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # verify web
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'worker')
-        self.assertEqual(container['release'], 'v3')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-worker-[0-9]{1,10}-[a-z0-9]{5}')
-
-        # verify web is still there
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        self.assertEqual(
+            set([item["type"] for item in response.data['results']]),
+            set(["web", "task", "worker"]),
+        )
+        self.assertPodContains(response.data['results'], app_id, 'web', "v3", "up")
+
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v3')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertEqual(response.json()['structure'], {'web': 1, 'task': 1, 'worker': 1})
+
+        # clean worker, repicas 1 not allow
+        response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id), {"ptypes": "task"})
+        self.assertEqual(response.status_code, 400, response.data)
+
+        # scale task
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'task': 0}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # clean worker, repicas 0 allow
+        response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id), {"ptypes": "task"})
+        self.assertEqual(response.status_code, 204, response.data)
 
         # look at the app structure
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.json()['structure'], {'web': 1, 'worker': 1})
 
-        # scale worker to make sure no info was lost
-        url = "/v2/apps/{app_id}/scale".format(**locals())
-        body = {'worker': 2}  # bump from 1 to 2
+        # ptypes is a required field
+        response = self.client.post('/v2/apps/{}/ptypes/clean'.format(app_id))
+        self.assertEqual(response.status_code, 400, response.data)
+
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'worker': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        # verify worker info
-        url = "/v2/apps/{app_id}/pods/worker".format(**locals())
+        # clean worker
+        response = self.client.post(
+            '/v2/apps/{}/ptypes/clean'.format(app_id),
+            {"ptypes": "worker"})
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'worker')
-        self.assertEqual(container['release'], 'v3')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-worker-[0-9]{1,10}-[a-z0-9]{5}')
-
-    @override_settings(DRYCC_DEPLOY_REJECT_IF_PROCFILE_MISSING=True)
-    def test_build_forgotten_procfile_reject(self, mock_requests):
-        """
-        Test that when a user first posts a build with a Procfile
-        and then later without it that missing process are actually
-        scaled down
-        """
-        # start with a new app
-        app_id = self.create_app()
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
-        # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
-        body = {
-            'image': 'autotest/example',
-            'stack': 'heroku-18',
-            'sha': 'a'*40,
-            'procfile': {
-                'web': 'node server.js',
-                'worker': 'node worker.js'
-            }
-        }
+        # do another deploy for this time forget Procfile
+        url = f"/v2/apps/{app_id}/build"
+        body = {'image': 'autotest/example:v11', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        # scale worker
-        url = "/v2/apps/{app_id}/scale".format(**locals())
-        body = {'worker': 1}
-        response = self.client.post(url, body)
+        # deploy
+        response = self.client.post(f"/v2/apps/{app_id}/releases/deploy/")
         self.assertEqual(response.status_code, 204, response.data)
 
-        # verify worker
-        url = "/v2/apps/{app_id}/pods/worker".format(**locals())
+        # look at the app structure
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'worker')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-worker-[0-9]{1,10}-[a-z0-9]{5}')
-
-        # do another deploy for this time forget Procfile
-        url = "/v2/apps/{app_id}/builds".format(**locals())
-        body = {'image': 'autotest/example', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 409, response.data)
+        self.assertEqual(response.json()['structure'], {'web': 1})
 
     def test_build_str(self, mock_requests):
         """Test the text representation of a build."""
         app_id = self.create_app()
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -464,14 +426,14 @@ def test_admin_can_create_builds_on_other_apps(self, mock_requests):
         """
         # create app as non-admin
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
 
         app_id = self.create_app()
 
         # post a new build as admin
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -490,9 +452,9 @@ def test_unauthorized_user_cannot_modify_build(self, mock_requests):
         app_id = self.create_app()
 
         unauthorized_user = User.objects.get(username='autotest2')
-        unauthorized_token = Token.objects.get(user=unauthorized_user).key
+        unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
-        url = '/v2/apps/{}/builds'.format(app_id)
+        url = '/v2/apps/{}/build'.format(app_id)
         body = {'image': 'foo'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
@@ -505,7 +467,7 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -518,19 +480,19 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
+        self.assertPodContains(response.data['results'], app_id, 'web', "v2", "up")
 
         # scale to zero
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
         # post another build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -542,56 +504,68 @@ def test_new_build_does_not_scale_up_automatically(self, mock_requests):
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
-    def test_build_image_in_registry(self, mock_requests):
+    def test_build_image_in_registry_ok(self, mock_requests):
         """When the image is already in the drycc registry no pull/tag/push happens"""
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
-        url = "/v2/apps/{app_id}/builds".format(**locals())
-        image = '127.0.0.1:5000/autotest/example'
+        url = f"/v2/apps/{app_id}/build"
+        image = 'registry.drycc.cc:5000/autotest/example'
         body = {'image': image, 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         build = Build.objects.get(uuid=response.data['uuid'])
         release = build.app.release_set.latest()
-        self.assertEqual(release.image, image)
+        self.assertEqual(release.get_deploy_image(PTYPE_WEB), image)
 
         # post an image as a build using registry hostname + port
-        url = "/v2/apps/{app_id}/builds".format(**locals())
-        image = '127.0.0.1:5000/autotest/example'
+        url = f"/v2/apps/{app_id}/build"
+        image = 'registry.drycc.cc:5000/autotest/example'
         body = {'image': image, 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         build = Build.objects.get(uuid=response.data['uuid'])
         release = build.app.release_set.latest()
-        self.assertEqual(release.image, image)
+        self.assertEqual(release.get_deploy_image(PTYPE_WEB), image)
+
+    def test_build_image_in_registry_err(self, mock_requests):
+        """When the image is already in the drycc registry no pull/tag/push happens"""
+        app_id = self.create_app()
+
+        # post an image as a build using registry hostname
+        url = f"/v2/apps/{app_id}/build"
+        for host in ['127.0.0.1', 'localhost', 'localhost:5000', '127.0.0.1:5000']:
+            image = f'{host}/autotest/example'
+            body = {'image': image, 'stack': 'container'}
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 400, response.data)
 
     def test_build_image_in_registry_with_auth(self, mock_requests):
         """add authentication to the build"""
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         image = 'autotest/example'
         response = self.client.post(url, {'image': image, 'stack': 'container'})
         self.assertEqual(response.status_code, 201, response.data)
 
         # add the required PORT information
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'PORT': '80'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {'values': [{"name": "PORT", "group": "global", "value": "80"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # set some registry information
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'registry': json.dumps({'username': 'bob', 'password': 'zoomzoom'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {'registry': {'web': {'username': 'bob', 'password': 'zoomzoom'}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -600,16 +574,16 @@ def test_build_image_in_registry_with_auth_no_port(self, mock_requests):
         app_id = self.create_app()
 
         # post an image as a build using registry hostname
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         image = 'autotest/example'
         response = self.client.post(url, {'image': image, 'stack': 'container'})
         self.assertEqual(response.status_code, 201, response.data)
 
         # set some registry information
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'registry': json.dumps({'username': 'bob', 'password': 'zoomzoom'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {'registry': json.dumps({'web': {'username': 'bob', 'password': 'zoomzoom'}})}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(response.status_code, 201, response.data)
 
     def test_release_create_failure(self, mock_requests):
         """
@@ -618,7 +592,7 @@ def test_release_create_failure(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app to get a build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -627,7 +601,7 @@ def test_release_create_failure(self, mock_requests):
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
 
-            url = "/v2/apps/{app_id}/builds".format(**locals())
+            url = f"/v2/apps/{app_id}/build"
             body = {'image': 'autotest/example'}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
@@ -641,19 +615,19 @@ def test_build_deploy_kube_failure(self, mock_requests):
         with mock.patch('scheduler.KubeHTTPClient.deploy') as mock_deploy:
             mock_deploy.side_effect = KubeException('Boom!')
 
-            url = "/v2/apps/{app_id}/builds".format(**locals())
+            url = f"/v2/apps/{app_id}/build"
             body = {'image': 'autotest/example', 'stack': 'container'}
             response = self.client.post(url, body)
-            exp_error = {'detail': '(app::deploy): Boom!'}
-            self.assertEqual(response.data, exp_error, response.data)
-            self.assertEqual(response.status_code, 400, response.data)
+            self.assertEqual(response.status_code, 201, response.data)
+            data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
+            self.assertEqual(data["results"][0]["state"], "crashed", data)
 
     def test_build_failures(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
 
         # deploy app to get a build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -663,18 +637,16 @@ def test_build_failures(self, mock_requests):
         # create a failed build to check that failed release is created
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
-
-            url = "/v2/apps/{app_id}/builds".format(**locals())
-            body = {'image': 'autotest/example', 'stack': 'container'}
+            url = f"/v2/apps/{app_id}/build"
+            body = {'image': 'autotest/example:v100', 'stack': 'container'}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400, response.data)
-            self.assertEqual(app.release_set.latest().version, 3)
-            self.assertEqual(app.release_set.filter(failed=False).latest().version, 2)
+            self.assertEqual(response.status_code, 201, response.data)
+            data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
+            self.assertEqual(data["results"][0]["state"], "crashed", data)
 
         # create a config to see that the new release is created with the last successful build
-        url = "/v2/apps/{app_id}/config".format(**locals())
-
-        body = {'values': json.dumps({'Test': 'test'})}
+        url = f"/v2/apps/{app_id}/config"
+        body = {'values': [{"name": "Test", "group": "global", "value": "test"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(app.release_set.latest().version, 4)
@@ -685,7 +657,7 @@ def test_build_validate_procfile(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app with incorrect proctype
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -698,7 +670,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -711,7 +683,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -724,7 +696,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -736,7 +708,7 @@ def test_build_validate_procfile(self, mock_requests):
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -749,7 +721,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
         # deploy app with empty command
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -762,7 +734,7 @@ def test_build_validate_procfile(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -774,3 +746,137 @@ def test_build_validate_procfile(self, mock_requests):
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
+
+    def test_dryccfile_ok(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/build"
+        default_image = "autotest/example"
+        run_image = "127.0.0.1:7070/myapp/run:git-123fsa1"
+        web_image = "127.0.0.1:7070/myapp/web:git-123fsa1"
+        worker_image = "127.0.0.1:7070/myapp/worker:git-123fsa1"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            "dryccfile": {
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": "127.0.0.1:7070/myapp/web:git-123fsa1",
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["python myworker.py"],
+                            "image": "127.0.0.1:7070/myapp/worker:git-123fsa1"
+                        },
+                    },
+                },
+            },
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
+            self.assertEqual(data["results"][0]["state"], "succeed", data)
+            app = App.objects.get(id=app_id)
+            for pod in app.list_pods():
+                if pod['type'] == 'run':
+                    self.assertEqual(pod["state"], "down", pod)
+                elif pod['type'] == 'web':
+                    self.assertEqual(pod["state"], "up", pod)
+            release = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()["results"][0]
+            self.assertEqual(release["state"], "succeed", data)
+            self.assertEqual(release["version"], 2, data)
+            release_obj = app.release_set.filter(version=release["version"])[0]
+            runners = release_obj.get_runners(["web"])
+            self.assertEqual(len(runners), 1)
+            self.assertEqual(runners[0]["image"], run_image, data)
+            self.assertEqual(release_obj.get_deploy_image("web"), web_image, data)
+            self.assertEqual(release_obj.get_deploy_image("worker"), worker_image, data)
+            self.assertEqual(release_obj.get_deploy_image("noexist"), default_image, data)
+
+    def test_dryccfile_format(self, mock_requests):
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'dryccfile': {
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                    },
+                },
+            },
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            app_id = self.create_app()
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 400, response.data)
+            body['dryccfile']['pipeline']["web.yaml"]['deploy'] = {'image': "127.0.0.1/cat/cat"}
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 400, response.data)
+            body['dryccfile']['pipeline']["worker.yaml"]['deploy'] = {'image': "127.0.0.1/cat/cat"}
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            body['dryccfile']['pipeline']["web.yaml"]['run'] = {
+                'command': ["bash", "-c"], 'args': ["ls /"]}
+            body['dryccfile']['pipeline']["worker.yaml"]['run'] = {
+                'command': ["bash", "-c"], 'args': ["ls /"]}
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            body['dryccfile']['pipeline']["web.yaml"] = {}
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 400, response.data)
+            body['dryccfile'] = {}
+            body['procfile'] = {
+                'web': 'node server.js',
+                'worker-test1': 'node worker.js'
+            }
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_canary.py b/rootfs/api/tests/test_canary.py
deleted file mode 100644
index a7139b550..000000000
--- a/rootfs/api/tests/test_canary.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import json
-from django.contrib.auth import get_user_model
-from django.core.cache import cache
-from rest_framework.authtoken.models import Token
-
-from api.models.app import App
-from api.tests import adapter, DryccTransactionTestCase
-import requests_mock
-
-User = get_user_model()
-
-
-@requests_mock.Mocker(real_http=True, adapter=adapter)
-class CanaryTest(DryccTransactionTestCase):
-
-    """Tests push notification from build system"""
-
-    fixtures = ['tests.json']
-
-    def setUp(self):
-        self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-    def tearDown(self):
-        # make sure every test has a clean slate for k8s mocking
-        cache.clear()
-
-    def pre_data(self):
-        app_id = self.create_app()
-        # check that updating config rolls a new release
-        url = f'/v2/apps/{app_id}/config'
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
-        # check that updating the build rolls a new release
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:v1', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['image'], body['image'])
-        # create v2 release
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:v2', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['image'], body['image'])
-        # get deployments
-        app = App.objects.get(id=app_id)
-        response = app._scheduler.deployments.get(app_id)
-        self.assertEqual(len(response.json()["items"]), 1)
-        # add canary
-        self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': ["web"]}
-        )
-        # add v3 release
-        url = f'/v2/apps/{app_id}/builds'
-        body = {'image': 'autotest/example:v3', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertEqual(response.data['image'], body['image'])
-        return app
-
-    def test_release(self, mock_requests):
-        app = self.pre_data()
-        # get deployments
-        response = app._scheduler.deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 2)
-        response = self.client.post(f'/v2/apps/{app.id}/canary/release/')
-        self.assertEqual(response.status_code, 201)
-        response = app._scheduler.deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 1)
-        self.assertEqual(
-            response.json()["items"][0]["spec"]["template"]["spec"]["containers"][0]["image"],
-            'autotest/example:v3',
-        )
-
-    def test_rollback(self, mock_requests):
-        app = self.pre_data()
-        response = app._scheduler.deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 2)
-        response = self.client.post(f'/v2/apps/{app.id}/canary/rollback/')
-        self.assertEqual(response.status_code, 201)
-        response = app._scheduler.deployments.get(app.id)
-        self.assertEqual(len(response.json()["items"]), 1)
-        self.assertEqual(
-            response.json()["items"][0]["spec"]["template"]["spec"]["containers"][0]["image"],
-            'autotest/example:v2',
-        )
diff --git a/rootfs/api/tests/test_certificate.py b/rootfs/api/tests/test_certificate.py
index 6f4724d0b..ccb9d5446 100644
--- a/rootfs/api/tests/test_certificate.py
+++ b/rootfs/api/tests/test_certificate.py
@@ -1,10 +1,16 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 from django.core.exceptions import SuspiciousOperation
 
+from api.models.app import App
 from api.models.certificate import Certificate
 from api.tests import TEST_ROOT, DryccTestCase
+from cryptography import x509
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from cryptography.x509.oid import NameOID
+from datetime import datetime, timedelta
+from pytz import utc
 
 User = get_user_model()
 
@@ -17,10 +23,11 @@ class CertificateTest(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-        self.url = '/v2/certs'
+        app_id = self.create_app(name='test-app-use-case')
+        self.app = App.objects.get(id=app_id)
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domain = 'autotest.example.com'
 
         with open('{}/certs/{}.key'.format(TEST_ROOT, self.domain)) as f:
@@ -33,6 +40,34 @@ def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
+    def _create_certificate_pem(self, common_name, san_names=None):
+        private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+        subject = issuer = x509.Name([
+            x509.NameAttribute(NameOID.COUNTRY_NAME, 'US'),
+            x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'Drycc'),
+            x509.NameAttribute(NameOID.COMMON_NAME, common_name),
+        ])
+        builder = x509.CertificateBuilder().subject_name(subject).issuer_name(issuer)
+        builder = builder.public_key(private_key.public_key())
+        builder = builder.serial_number(x509.random_serial_number())
+        builder = builder.not_valid_before(datetime.now(utc) - timedelta(minutes=1))
+        builder = builder.not_valid_after(datetime.now(utc) + timedelta(days=30))
+
+        if san_names:
+            builder = builder.add_extension(
+                x509.SubjectAlternativeName([x509.DNSName(name) for name in san_names]),
+                critical=False,
+            )
+
+        certificate = builder.sign(private_key=private_key, algorithm=hashes.SHA256())
+        cert_pem = certificate.public_bytes(serialization.Encoding.PEM).decode('utf-8')
+        key_pem = private_key.private_bytes(
+            encoding=serialization.Encoding.PEM,
+            format=serialization.PrivateFormat.TraditionalOpenSSL,
+            encryption_algorithm=serialization.NoEncryption(),
+        ).decode('utf-8')
+        return cert_pem, key_pem, certificate
+
     def test_create_certificate_with_domain(self):
         """Tests creating a certificate."""
         response = self.client.post(
@@ -145,17 +180,17 @@ def test_delete_certificate(self):
         """Destroying a certificate should generate a 204 response"""
         Certificate.objects.create(
             name='random-test-cert',
-            owner=self.user,
+            app=self.app,
             common_name='autotest.example.com',
             certificate=self.cert,
             key=self.key
         )
-        url = '/v2/certs/random-test-cert'
+        url = f'/v2/apps/{self.app.id}/certs/random-test-cert'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
 
     def test_create_invalid_cert(self):
-        """Upload a cert that can't be loaded by pyopenssl"""
+        """Upload a cert that can't be parsed."""
         response = self.client.post(
             self.url,
             {
@@ -165,22 +200,22 @@ def test_create_invalid_cert(self):
             }
         )
         self.assertEqual(response.status_code, 400, response.data)
-        # match partial since right now the rest is pyopenssl errors
+        # Match partial since parser details may vary.
         self.assertIn('Could not load certificate', response.data['certificate'][0])
 
     def test_load_invalid_cert(self):
-        """Inject a cert that can't be loaded by pyopenssl"""
+        """Inject a cert that can't be parsed."""
 
         with self.assertRaises(SuspiciousOperation):
             Certificate.objects.create(
-                owner=self.user,
+                app=self.app,
                 name='random-test-cert',
                 certificate='i am bad data',
                 key='i am bad data as well'
             )
 
     def test_create_invalid_key(self):
-        """Upload a private key that can't be loaded by pyopenssl"""
+        """Upload a private key that can't be parsed."""
         response = self.client.post(
             self.url,
             {
@@ -190,20 +225,71 @@ def test_create_invalid_key(self):
             }
         )
         self.assertEqual(response.status_code, 400, response.data)
-        # match partial since right now the rest is pyopenssl errors
+        # Match partial since parser details may vary.
         self.assertIn('Could not load private key', response.data['key'][0])
 
     def test_load_invalid_key(self):
-        """Inject a private key that can't be loaded by pyopenssl"""
+        """Inject a private key that can't be parsed."""
 
         with self.assertRaises(SuspiciousOperation):
             Certificate.objects.create(
-                owner=self.user,
+                app=self.app,
                 name='random-test-cert',
                 certificate=self.cert,
                 key='I am Groot.'
             )
 
+    def test_create_certificate_with_san(self):
+        """Certificates with SAN should expose DNS names."""
+        cert, key, _ = self._create_certificate_pem(
+            'autotest.example.com',
+            san_names=['autotest.example.com', 'www.autotest.example.com'],
+        )
+
+        response = self.client.post(
+            self.url,
+            {
+                'name': 'san-test-cert',
+                'certificate': cert,
+                'key': key,
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(
+            response.data['san'],
+            ['autotest.example.com', 'www.autotest.example.com'],
+        )
+
+    def test_create_certificate_with_mismatched_key(self):
+        """Certificate and private key must match."""
+        cert, _, _ = self._create_certificate_pem('autotest.example.com')
+        _, other_key, _ = self._create_certificate_pem('other.example.com')
+
+        response = self.client.post(
+            self.url,
+            {
+                'name': 'mismatch-test-cert',
+                'certificate': cert,
+                'key': other_key,
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertIn('Certificate and private key do not match!', response.data[0])
+
+    def test_certificate_persists_subject_and_issuer(self):
+        """Persisted certificate metadata should reflect x509 subject and issuer."""
+        cert, key, parsed = self._create_certificate_pem('autotest.example.com')
+
+        certificate = Certificate.objects.create(
+            name='metadata-test-cert',
+            app=self.app,
+            certificate=cert,
+            key=key,
+        )
+
+        self.assertEqual(certificate.subject, parsed.subject.rfc4514_string())
+        self.assertEqual(certificate.issuer, parsed.issuer.rfc4514_string())
+
     def test_certs_fetch_limit(self):
         """
         When a user retrieves a certificate, make sure limits work
diff --git a/rootfs/api/tests/test_certificate_use_case_1.py b/rootfs/api/tests/test_certificate_use_case_1.py
index ef9c260ef..604a9f4b4 100644
--- a/rootfs/api/tests/test_certificate_use_case_1.py
+++ b/rootfs/api/tests/test_certificate_use_case_1.py
@@ -1,8 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -21,12 +21,14 @@ class CertificateUseCase1Test(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-1')
-        self.domain = Domain.objects.create(owner=self.user, app=self.app, domain='foo.com')
+        app_id = self.create_app(name='test-app-use-case-1')
+        self.app = App.objects.get(id=app_id)
+        self.url = f'/v2/apps/{self.app.id}/certs'
+        self.domain = Domain.objects.create(
+            app=self.app, domain='foo.com', ptype=PTYPE_WEB)
         self.name = 'foo-com'  # certificate name
 
         with open('{}/certs/{}.key'.format(TEST_ROOT, self.domain)) as f:
@@ -146,13 +148,13 @@ def test_certficate_denied_requests(self):
     def test_delete_certificate(self):
         """Destroying a certificate should generate a 204 response"""
         Certificate.objects.create(
-            owner=self.user,
+            app=self.app,
             name=self.name,
             certificate=self.cert,
             key=self.key
         )
 
-        url = '/v2/certs/{}'.format(self.name)
+        url = f'/v2/apps/{self.app.id}/certs/{self.name}/'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
 
@@ -189,7 +191,7 @@ def test_delete_certificate_with_attached_domain(self):
         self.assertEqual(domain.certificate.name, self.name)
 
         # Delete certificate
-        url = '/v2/certs/{}'.format(self.name)
+        url = f'/v2/apps/{self.app.id}/certs/{self.name}/'
         response = self.client.delete(url)
         self.assertEqual(response.status_code, 204, response.data)
 
diff --git a/rootfs/api/tests/test_certificate_use_case_2.py b/rootfs/api/tests/test_certificate_use_case_2.py
index 9a062ecb9..c74845291 100644
--- a/rootfs/api/tests/test_certificate_use_case_2.py
+++ b/rootfs/api/tests/test_certificate_use_case_2.py
@@ -1,8 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -21,14 +21,17 @@ class CertificateUseCase2Test(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-2')
+        app_id = self.create_app(name='test-app-use-case-2')
+        self.app = App.objects.get(id=app_id)
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
-            'foo.com': Domain.objects.create(owner=self.user, app=self.app, domain='foo.com'),
-            'bar.com': Domain.objects.create(owner=self.user, app=self.app, domain='bar.com'),
+            'foo.com': Domain.objects.create(
+                app=self.app, domain='foo.com', ptype=PTYPE_WEB),
+            'bar.com': Domain.objects.create(
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         # only foo.com has a cert
@@ -130,11 +133,11 @@ def test_delete_certificate(self):
         for domain, certificate in self.certificates.items():
             Certificate.objects.create(
                 name=certificate['name'],
-                owner=self.user,
+                app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_certificate_use_case_3.py b/rootfs/api/tests/test_certificate_use_case_3.py
index 6988e8f03..7eedfc859 100644
--- a/rootfs/api/tests/test_certificate_use_case_3.py
+++ b/rootfs/api/tests/test_certificate_use_case_3.py
@@ -1,8 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -21,14 +21,17 @@ class CertificateUseCase3Test(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-3')
+        app_id = self.create_app(name='test-app-use-case-3')
+        self.app = App.objects.get(id=app_id)
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
-            'foo.com': Domain.objects.create(owner=self.user, app=self.app, domain='foo.com'),
-            'bar.com': Domain.objects.create(owner=self.user, app=self.app, domain='bar.com'),
+            'foo.com': Domain.objects.create(
+                app=self.app, domain='foo.com', ptype=PTYPE_WEB),
+            'bar.com': Domain.objects.create(
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
@@ -132,11 +135,11 @@ def test_delete_certificate(self):
         for domain, certificate in self.certificates.items():
             Certificate.objects.create(
                 name=certificate['name'],
-                owner=self.user,
+                app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_certificate_use_case_4.py b/rootfs/api/tests/test_certificate_use_case_4.py
index 7ca227d4b..573cfa571 100644
--- a/rootfs/api/tests/test_certificate_use_case_4.py
+++ b/rootfs/api/tests/test_certificate_use_case_4.py
@@ -1,8 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -21,15 +21,20 @@ class CertificateUseCase4Test(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-3')
+        app_id = self.create_app(name='test-app-use-case-3')
+        self.app = App.objects.get(id=app_id)
+        self.url = f'/v2/apps/{self.app.id}/certs'
         self.domains = {
-            '*.foo.com': Domain.objects.create(owner=self.user, app=self.app, domain='*.foo.com'),
-            'foo.com': Domain.objects.create(owner=self.user, app=self.app, domain='foo.com'),
-            'bar.com': Domain.objects.create(owner=self.user, app=self.app, domain='bar.com'),
+            '*.foo.com': Domain.objects.create(
+                app=self.app, domain='*.foo.com',
+                ptype=PTYPE_WEB),
+            'foo.com': Domain.objects.create(
+                app=self.app, domain='foo.com', ptype=PTYPE_WEB),
+            'bar.com': Domain.objects.create(
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
@@ -164,8 +169,8 @@ def test_certificate_attach_overwrite(self):
             '{}/{}/domain/'.format(self.url, 'bar-com'),
             {'domain': 'foo.com'}
         )
-        # Should be a 409 Conflict since it already existed
-        self.assertEqual(response.status_code, 409)
+        # allow re attach to update
+        self.assertEqual(response.status_code, 201)
 
         # Assert that domain and cert are still the original
         response = self.client.get(
@@ -176,7 +181,7 @@ def test_certificate_attach_overwrite(self):
         expected = {
             'name': 'foo-com',
             'common_name': 'foo.com',
-            'domains': ['foo.com']
+            'domains': []
         }
         for key, value in list(expected.items()):
             self.assertEqual(
@@ -193,7 +198,7 @@ def test_certificate_attach_overwrite(self):
         expected = {
             'name': 'bar-com',
             'common_name': 'bar.com',
-            'domains': []
+            'domains': ['foo.com']
         }
         for key, value in list(expected.items()):
             self.assertEqual(
@@ -214,11 +219,11 @@ def test_delete_certificate(self):
         for domain, certificate in self.certificates.items():
             Certificate.objects.create(
                 name=certificate['name'],
-                owner=self.user,
+                app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_certificate_use_case_5.py b/rootfs/api/tests/test_certificate_use_case_5.py
index 2ab6d68b9..1a060421b 100644
--- a/rootfs/api/tests/test_certificate_use_case_5.py
+++ b/rootfs/api/tests/test_certificate_use_case_5.py
@@ -1,8 +1,8 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
 from api.models.certificate import Certificate
 from api.models.domain import Domain
 from api.tests import TEST_ROOT, DryccTestCase
@@ -21,16 +21,21 @@ class CertificateUseCase5Test(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.url = '/v2/certs'
-        self.app = App.objects.create(owner=self.user, id='test-app-use-case-5')
+        app_id = self.create_app(name='test-app-use-case-5')
+        self.app = App.objects.get(id=app_id)
+        self.url = f'/v2/apps/{self.app.id}/certs'
         # Done out of scope as it gets the same cert as the wildcard
-        Domain.objects.create(owner=self.user, app=self.app, domain='foo.com')
+        Domain.objects.create(
+            app=self.app, domain='foo.com', ptype=PTYPE_WEB)
         self.domains = {
-            '*.foo.com': Domain.objects.create(owner=self.user, app=self.app, domain='*.foo.com'),
-            'bar.com': Domain.objects.create(owner=self.user, app=self.app, domain='bar.com'),
+            '*.foo.com': Domain.objects.create(
+                app=self.app, domain='*.foo.com',
+                ptype=PTYPE_WEB),
+            'bar.com': Domain.objects.create(
+                app=self.app, domain='bar.com', ptype=PTYPE_WEB),
         }
 
         self.certificates = {}
@@ -160,13 +165,13 @@ def test_delete_certificate(self):
             # Create certificate
             Certificate.objects.create(
                 name=certificate['name'],
-                owner=self.user,
+                app=self.app,
                 common_name=domain,
                 certificate=certificate['cert'],
                 key=certificate['key']
             )
 
             # Remove certificate
-            url = '/v2/certs/{}'.format(certificate['name'])
+            url = f'/v2/apps/{self.app.id}/certs/{certificate['name']}/'
             response = self.client.delete(url)
             self.assertEqual(response.status_code, 204, response.data)
diff --git a/rootfs/api/tests/test_config.py b/rootfs/api/tests/test_config.py
index 2c688bb72..c9579df42 100644
--- a/rootfs/api/tests/test_config.py
+++ b/rootfs/api/tests/test_config.py
@@ -4,16 +4,20 @@
 
 Run the tests with "./manage.py test api"
 """
-import json
-
+import copy
+from io import StringIO
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
+from django.core.management import call_command
+
 from unittest import mock
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_RUN, PTYPE_WEB
 from api.models.config import Config
+from api.serializers import CONFIG_LIMITS_MISMATCH_MSG
+from api.models.build import Build
+from api.models.release import Release
 
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
@@ -29,17 +33,12 @@ class ConfigTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-        url = '/v2/apps'
-        response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token))
-        self.assertEqual(response.status_code, 201, response.data)
-        self.app = App.objects.all()[0]
+        app_id = self.create_app()
+        self.app = App.objects.get(id=app_id)
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
@@ -51,210 +50,419 @@ def test_config(self, mock_requests):
         app_id = self.create_app()
 
         # check to see that an initial/empty config was created
-        url = "/v2/apps/{app_id}/config".format(**locals())
+        url = f"/v2/apps/{app_id}/config"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('values', response.data)
-        self.assertEqual(response.data['values'], {})
+        self.assertEqual(response.data['values'], [])
         config1 = response.data
 
         # set an initial config value
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        value1 = {"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         config2 = response.data
         self.assertNotEqual(config1['uuid'], config2['uuid'])
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
 
         # read the config
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         config3 = response.data
         self.assertEqual(config2, config3)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
 
         # set an additional config value
-        body = {'values': json.dumps({'NEW_URL2': 'http://localhost:8080/'})}
+        value2 = {"name": "NEW_URL2", "value": "http://localhost:8080/", "group": "global"}
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         config3 = response.data
         self.assertNotEqual(config2['uuid'], config3['uuid'])
-        self.assertIn('NEW_URL1', response.data['values'])
-        self.assertIn('NEW_URL2', response.data['values'])
+        self.assertEqual([value1, value2], response.data['values'])
 
         # read the config again
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         config4 = response.data
         self.assertEqual(config3, config4)
-        self.assertIn('NEW_URL1', response.data['values'])
-        self.assertIn('NEW_URL2', response.data['values'])
+        self.assertEqual([value1, value2], response.data['values'])
 
         # unset a config value
-        body = {'values': json.dumps({'NEW_URL2': None})}
+        value2['value'] = None
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         config5 = response.data
         self.assertNotEqual(config4['uuid'], config5['uuid'])
-        self.assertNotIn('NEW_URL2', json.dumps(response.data['values']))
+        self.assertEqual([value1], response.data['values'])
 
         # unset all config values
-        body = {'values': json.dumps({'NEW_URL1': None})}
+        value1['value'] = None
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertNotIn('NEW_URL1', json.dumps(response.data['values']))
+        self.assertEqual([], response.data['values'])
 
         # set a port and then unset it to make sure validation ignores the unset
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value3 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value3]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value3], response.data['values'])
 
-        body = {'values': json.dumps({'PORT': None})}
+        value3['value'] = None
+        body = {'values': [value3]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([], response.data['values'])
+        # repeat settings
+        value2['value'] = "VALUE2"
+        value3['value'] = None
+        body = {'values': [value2, value3]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertNotIn('PORT', response.data['values'])
+        self.assertEqual([value2], response.data['values'])
+        # set empty env
+        value1['value'] = ''
+        body = {'values': [value1]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([value2, value1], response.data['values'])
 
-        # disallow put/patch/delete
+        # disallow put/patch
         response = self.client.put(url)
         self.assertEqual(response.status_code, 405, response.data)
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
         return config5
 
-    def test_default_tags(self, mock_requests):
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = '{"ssd": "true"}'
+    def test_config_group_merge_false(self, mock_requests):
+        """
+        Test that config can be created with merge='false' parameter
+        which replaces the entire config instead of merging
+        """
         app_id = self.create_app()
-        url = "/v2/apps/{app_id}/config".format(**locals())
 
+        # set initial config values
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "group": "global"}
+        value2 = {"name": "DATABASE_URL", "value": "postgres://localhost/db", "group": "global"}
+        body = {'values': [value1, value2]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(len(response.data['values']), 2)
+
+        # replace entire config with merge='false'
+        value3 = {"name": "NEW_CONFIG", "value": "replacement_value", "group": "global"}
+        body = {'values': [value1, value2, value3]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
         response = self.client.get(url)
-        expected = {
-            'owner': self.user.username,
-            'app': app_id,
-            'values': {},
-            'memory': {},
-            'cpu': {},
-            'tags': {'ssd': 'true'},
-            'registry': {}
-        }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 3)
+
+        # remove value3
+        body = {'values': [value1, value2]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 2)
+
+    def test_config_ptype_merge_false(self, mock_requests):
+        """
+        Test that config can be created with merge='false' parameter
+        which replaces the entire config instead of merging
+        """
+        app_id = self.create_app()
+
+        # set initial config values
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "ptype": "web"}
+        value2 = {"name": "DATABASE_URL", "value": "postgres://localhost/db", "ptype": "web"}
+        value3 = {"name": "NEW_CONFIG", "value": "replacement_value", "ptype": "web"}
+        body = {'values': [value1, value2, value3]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 3)
+
+        # remove value3
+        body = {'values': [value1, value2]}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 201, response.data)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 2)
 
-        # make sure changes not drop tags
-        body = {'values': json.dumps({'PORT': '5001'})}
+    def test_config_merge_false_with_empty_values(self, mock_requests):
+        """
+        Test that config can be created with merge='false' and empty values
+        """
+        app_id = self.create_app()
+
+        # set initial config values
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
-        expected = {
-            'owner': self.user.username,
-            'app': app_id,
-            'values': {'PORT': '5001'},
-            'memory': {},
-            'cpu': {},
-            'tags': {'ssd': 'true'},
-            'registry': {}
-        }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(len(response.data['values']), 1)
+
+        # replace with empty config using merge='false'
+        body = {'values': []}
+        response = self.client.post(f"{url}?merge=false", body)
+        self.assertEqual(response.status_code, 409, response.data)
+
+        # verify config is empty
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 1)
+
+    def test_config_merge_false_case_insensitive(self, mock_requests):
+        """
+        Test that merge parameter is case insensitive
+        """
+        app_id = self.create_app()
+
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "DEBUG", "value": "true", "group": "global"}
+        body = {'values': [value1]}
+
+        # test with uppercase FALSE
+        response = self.client.post(f"{url}?merge=FALSE", body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # verify config was replaced
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['values']), 1)
+
+    def test_registry_set(self, mock_requests):
+        app_id = self.create_app()
+        # set an initial config value
+        url = f"/v2/apps/{app_id}/config"
+        web_registry = {'web': {'username': 'admin', 'password': 'admin'}}
+        body = {'registry': web_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data['registry'], web_registry, response.data)
+
+        task_registry = {'task': {'username': 'admin', 'password': 'admin'}}
+        body = {'registry': task_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        all_registry = {}
+        all_registry.update(web_registry)
+        all_registry.update(task_registry)
+        self.assertEqual(response.data['registry'], all_registry, response.data)
+
+        # delete task username
+        task_registry = {'task': {'username': None, 'password': 'admin'}}
+        body = {'registry': task_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        all_registry = {}
+        all_registry.update(web_registry)
+        all_registry.update(task_registry)
+        del all_registry['task']['username']
+        self.assertEqual(response.data['registry'], all_registry, response.data)
+
+        # delete task registry
+        task_registry = {'task': None}
+        body = {'registry': task_registry}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data['registry'], web_registry, response.data)
+
+    def test_values_refs(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+        values = [{"name": "DEBUG", "value": "true", "group": "global"}]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'true'})
+        self.assertEqual(config.envs("task"), {'DEBUG': 'true'})
+
+        # add test error, changed nothing
+        values = [
+            {"name": "DEBUG", "value": "true", "group": "global"},
+        ]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 409, response.data)
+
+        # add values ok
+        values = [
+            {"name": "DEBUG", "value": "true", "group": "mytask1"},
+            {"name": "APP", "value": "task2", "group": "mytask2"},
+            {"name": "DEBUG", "value": "false", "group": "global"},
+        ]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'false'})
+        self.assertEqual(config.envs("task"), {'DEBUG': 'false'})
+
+        # add values_refs
+        body = {'values_refs': {"task": ["mytask1", "mytask2"]}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'false'})
+        self.assertEqual(config.envs("task"), {'DEBUG': 'true', 'APP': 'task2'})
+
+        # add new item for mytask
+        values = [
+            {"name": "VERSION", "value": "1.0.1", "group": "mytask1"},
+        ]
+        body = {'values': values}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        self.assertEqual(config.envs("web"), {'DEBUG': 'false'})
+        self.assertEqual(
+            config.envs("task"), {'DEBUG': 'true', 'VERSION': '1.0.1', 'APP': 'task2'})
+
+        # remove tas values_refs
+        body = {'values_refs': {'task': ['mytask1']}}
+        response = self.client.delete(url, body)
+        self.assertEqual(response.status_code, 200, response.data)
+        url = '/v2/apps/{}/config'.format(app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data['values_refs'], {"task": ["mytask2"]}, response.data)
 
     def test_response_data(self, mock_requests):
         """Test that the serialized response contains only relevant data."""
         app_id = self.create_app()
 
-        url = "/v2/apps/{app_id}/config".format(**locals())
+        url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'memory',
-                                'cpu', 'tags', 'registry', 'healthcheck', 'lifecycle_post_start',
-                                'lifecycle_pre_stop', 'termination_grace_period'])
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'values',
+                                'values_refs', 'limits', 'tags', 'registry', 'healthcheck',
+                                'lifecycle', 'termination_grace_period'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
-            'values': {'PORT': '5000'},
-            'memory': {},
-            'cpu': {},
+            'values': [value1],
+            'limits': {
+                PTYPE_RUN: 'std1.large.c1m1',
+                PTYPE_WEB: 'std1.large.c1m1'
+            },
             'tags': {},
             'registry': {}
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, response.data | expected)
 
     def test_response_data_types_converted(self, mock_requests):
         """Test that config data is converted into the correct type."""
         app_id = self.create_app()
 
-        url = "/v2/apps/{app_id}/config".format(**locals())
-
-        body = {'values': json.dumps({'PORT': 5000}), 'cpu': json.dumps({'web': '1000m'})}
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {
+            'values': [value1],
+            'limits': {
+                PTYPE_WEB: 'std1.large.c1m2',
+            }
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'values', 'memory',
-                                'cpu', 'tags', 'registry', 'healthcheck', 'lifecycle_post_start',
-                                'lifecycle_pre_stop', 'termination_grace_period'])
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'values', 'limits',
+                                'values_refs', 'tags', 'registry', 'healthcheck',
+                                'lifecycle', 'termination_grace_period'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
-            'values': {'PORT': '5000'},
-            'memory': {'web': '1G'},
-            'cpu': {'web': "1000m"},
+            'values': [value1],
+            'limits': {
+                PTYPE_RUN: 'std1.large.c1m1',
+                PTYPE_WEB: 'std1.large.c1m2'
+            },
             'tags': {},
             'registry': {}
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, expected | response.data)
 
-        body = {'cpu': json.dumps({'web': 'this will fail'})}
+        body = {'limits': {PTYPE_WEB: "not-exist"}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        self.assertIn(
-            'CPU limit format: <value>, where value must be a numeric',
-            response.data['cpu'])
+        self.assertEqual(
+            str(response.data["limits"][0]),
+            CONFIG_LIMITS_MISMATCH_MSG.format("not-exist")
+        )
 
     def test_config_set_same_key(self, mock_requests):
         """
         Test that config sets on the same key function properly
         """
         app_id = self.create_app()
-        url = "/v2/apps/{app_id}/config".format(**locals())
+        url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
 
         # reset same config value
-        body = {'values': json.dumps({'PORT': '5001'})}
+        value1['value'] = '5001'
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
-        self.assertEqual(response.data['values']['PORT'], '5001')
+        self.assertEqual([value1], response.data['values'])
 
     def test_config_set_unicode(self, mock_requests):
         """
         Test that config sets with unicode values are accepted.
         """
         app_id = self.create_app()
-        url = "/v2/apps/{app_id}/config".format(**locals())
+        url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
-        body = {'values': json.dumps({'POWERED_BY': 'Деис'})}
+        value1 = {"name": "POWERED_BY", "value": "Деис", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('POWERED_BY', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         # reset same config value
-        body = {'values': json.dumps({'POWERED_BY': 'Кроликов'})}
+        value1['value'] = 'Кроликов'
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('POWERED_BY', response.data['values'])
-        self.assertEqual(response.data['values']['POWERED_BY'], 'Кроликов')
+        self.assertEqual([value1], response.data['values'])
 
         # set an integer to test unicode regression
-        body = {'values': json.dumps({'INTEGER': 1})}
+        value2 = {"name": "INTEGER", "value": "1", "group": "global"}
+        body = {'values': [value2]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('INTEGER', response.data['values'])
-        self.assertEqual(response.data['values']['INTEGER'], '1')
+        self.assertEqual([value1, value2], response.data['values'])
 
     def test_config_str(self, mock_requests):
         """Test the text representation of a node."""
@@ -267,12 +475,15 @@ def test_valid_config_keys(self, mock_requests):
         """
         keys = ("FOO", "_foo", "f001", "FOO_BAR_BAZ_")
         app_id = self.create_app()
-        url = '/v2/apps/{app_id}/config'.format(**locals())
+        url = f'/v2/apps/{app_id}/config'
+        values = []
         for k in keys:
-            body = {'values': json.dumps({k: "testvalue"})}
+            value1 = {"name": k, "value": "testvalue", "group": "global"}
+            values.append(value1)
+            body = {'values': [value1]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201)
-            self.assertIn(k, response.data['values'])
+            self.assertEqual(values, response.data['values'])
 
     def test_config_deploy_failure(self, mock_requests):
         """
@@ -281,7 +492,7 @@ def test_config_deploy_failure(self, mock_requests):
         app_id = self.create_app()
 
         # deploy app to get a build
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -289,19 +500,25 @@ def test_config_deploy_failure(self, mock_requests):
 
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
-            url = '/v2/apps/{app_id}/config'.format(**locals())
-            body = {'values': json.dumps({'test': "testvalue"})}
+            url = f'/v2/apps/{app_id}/config'
+            value1 = {"name": "test", "value": "testvalue", "group": "global"}
+            body = {'values': [value1]}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400)
+            self.assertEqual(response.status_code, 201)
+            app = App.objects.get(id=app_id)
+            release = app.release_set.latest()
+            self.assertEqual(release.failed, True)
+            self.assertEqual(release.conditions[0]['exception'], "Exception: Boom!")
 
     def test_invalid_config_keys(self, mock_requests):
         """Test that invalid config keys are rejected.
         """
-        keys = ("123", "../../foo", "FOO/", "FOO-BAR")
+        keys = ("$123", "../../foo", "FOO/", "*FOO-BAR")
         app_id = self.create_app()
-        url = '/v2/apps/{app_id}/config'.format(**locals())
+        url = f'/v2/apps/{app_id}/config'
         for k in keys:
-            body = {'values': json.dumps({k: "testvalue"})}
+            value1 = {"name": k, "value": "testvalue", "group": "global"}
+            body = {'values': [value1]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400)
 
@@ -311,13 +528,13 @@ def test_invalid_config_values(self, mock_requests):
         Right now only PORT is checked
         """
         data = [
-            {'field': 'PORT', 'value': 'dog'},
-            {'field': 'PORT', 'value': 99999}
+            {"name": "PORT", "value": "dog", "group": "global"},
+            {"name": "PORT", "value": "99999", "group": "global"}
         ]
         app_id = self.create_app()
-        url = '/v2/apps/{app_id}/config'.format(**locals())
+        url = f'/v2/apps/{app_id}/config'
         for row in data:
-            body = {'values': json.dumps({row['field']: row['value']})}
+            body = {'values': [row]}
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 400, response.data)
 
@@ -326,28 +543,21 @@ def test_admin_can_create_config_on_other_apps(self, mock_requests):
         values for that app.
         """
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
 
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
         app_id = self.create_app()
-        url = "/v2/apps/{app_id}/config".format(**locals())
+        url = f"/v2/apps/{app_id}/config"
 
         # set an initial config value
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        body = {'values': json.dumps({'PORT': '5000'})}
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('PORT', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         return response
 
-    def test_config_owner_is_requesting_user(self, mock_requests):
-        """
-        Ensure that setting the config value is owned by the requesting user
-        See https://github.com/drycc/drycc/issues/2650
-        """
-        response = self.test_admin_can_create_config_on_other_apps()
-        self.assertEqual(response.data['owner'], self.user.username)
-
     def test_unauthorized_user_cannot_modify_config(self, mock_requests):
         """
         An unauthorized user should not be able to modify other config.
@@ -358,10 +568,11 @@ def test_unauthorized_user_cannot_modify_config(self, mock_requests):
         app_id = self.create_app()
 
         unauthorized_user = User.objects.get(username='autotest2')
-        unauthorized_token = Token.objects.get(user=unauthorized_user).key
+        unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
         url = '/v2/apps/{}/config'.format(app_id)
-        body = {'values': {'FOO': 'bar'}}
+        value1 = {"name": "FOO", "value": "bar", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
 
@@ -376,32 +587,37 @@ def test_config_failures(self, mock_requests):
         app = App.objects.get(id=app_id)
 
         # deploy app to get a build
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(response.data['image'], body['image'])
 
         # set an initial config value
-        url = "/v2/apps/{app_id}/config".format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}
+        body = {'values': [value1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual([value1], response.data['values'])
         success_config = app.release_set.latest().config
 
         # create a failed config to check that failed release is created
         with mock.patch('api.models.app.App.deploy') as mock_deploy:
             mock_deploy.side_effect = Exception('Boom!')
-            url = '/v2/apps/{app_id}/config'.format(**locals())
-            body = {'values': json.dumps({'test': "testvalue"})}
+            url = f'/v2/apps/{app_id}/config'
+            value1 = {"name": "test", "value": "testvalue", "group": "global"}
+            body = {'values': [value1]}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400)
+            self.assertEqual(response.status_code, 201)
             self.assertEqual(app.release_set.latest().version, 4)
+            self.assertEqual(app.release_set.latest().failed, True)
+            self.assertEqual(
+                app.release_set.latest().conditions[0]['exception'], "Exception: Boom!")
             self.assertEqual(app.release_set.filter(failed=False).latest().version, 3)
 
         # create a build to see that the new release is created with the last successful config
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -409,12 +625,100 @@ def test_config_failures(self, mock_requests):
         self.assertEqual(app.release_set.latest().config, success_config)
         self.assertEqual(app.config_set.count(), 3)
 
+    def test_unset_limits(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {
+            'values': [value1],
+            'limits': {
+                "task": 'std1.large.c2m4',
+                PTYPE_RUN: 'std1.large.c2m4',
+                PTYPE_WEB: 'std1.large.c2m4',
+            },
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201)
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data["limits"], body["limits"])
+        # unset ok
+        body = {
+            'limits': {
+                  "task": None,
+            },
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201)
+        response = self.client.get(url)
+        self.assertEqual(
+            response.data["limits"],
+            {PTYPE_RUN: 'std1.large.c2m4', PTYPE_WEB: 'std1.large.c2m4'},
+        )
+
+    def test_unset_limits_error(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {
+            'values': [value1],
+            'limits': {
+                "task": 'std1.large.c2m4',
+                PTYPE_RUN: 'std1.large.c2m4',
+                PTYPE_WEB: 'std1.large.c2m4',
+            },
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201)
+        # dockerfile + procfile worflow
+        app = App.objects.get(id=app_id)
+        build = Build.objects.create(
+            app=app,
+            image="qwerty",
+            procfile={
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            },
+            dockerfile='foo',
+            sha='somereallylongsha'
+        )
+        # create an initial release
+        release = Release.objects.create(
+            version=3,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        # deploy
+        release.deploy()
+        # unset error
+        body = {
+            'limits': {
+                  "no-exists": None,
+            },
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 422)
+        self.assertEqual(str(response.data["detail"]), "no-exists does not exist under limits")
+        # scale up
+        body = {'web': 3}
+        response = self.client.post(f"/v2/apps/{app_id}/ptypes/scale", body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        body = {
+            'limits': {
+                  PTYPE_WEB: None,
+            },
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 422)
+        self.assertEqual(
+            str(response.data["detail"]), "the web has already been used and cannot be deleted")
+
     def call_command(self, *args, **kwargs):
-        from io import StringIO
-        from django.core.management import call_command
         out = StringIO()
         call_command(
-            "measure_apps",
+            "upload_app_usage",
             *args,
             stdout=out,
             stderr=StringIO(),
@@ -426,7 +730,364 @@ def test_measure_config(self, *args, **kwargs):
         # create
         app_id = self.create_app()
         url = f"/v2/apps/{app_id}/config"
-        body = {'values': json.dumps({'PORT': 5000}), 'cpu': json.dumps({'web': '1000m'})}
-        self.client.post(url, body)
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {
+            'values': [value1],
+            'limits': {
+                  PTYPE_RUN: 'std1.large.c2m4',
+                  PTYPE_WEB: 'std1.large.c2m4',
+            },
+        }
+        response = self.client.post(url, body)
         out = self.call_command()
         self.assertIn(out, "done\n")
+        self.assertEqual(response.status_code, 201)
+
+    def test_set_config_limits_run(self, *args, **kwargs):
+        # create
+        app_id = self.create_app()
+        # dockerfile + procfile worflow
+        app = App.objects.get(id=app_id)
+        build = Build.objects.create(
+            app=app,
+            image="qwerty",
+            procfile={
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            },
+            dockerfile='foo',
+            sha='somereallylongsha'
+        )
+        # create an initial release
+        release = Release.objects.create(
+            version=3,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        # deploy
+        release.deploy()
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {
+            'values': [value1],
+            'limits': {
+                  PTYPE_RUN: 'std1.large.c2m4',
+                  PTYPE_WEB: 'std1.large.c2m4',
+            },
+        }
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        expect = {'run': 'std1.large.c2m4', 'web': 'std1.large.c2m4', 'worker': 'std1.large.c1m1'}
+        self.assertEqual(expect, response.json()["limits"], response.data)
+
+    def test_config_set_typed_values(self, mock_requests):
+        """
+        Test that config sets on the same key function properly
+        """
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+
+        # set an initial config value
+        body = {'values': [{"name": "PORT", "value": "5000", "ptype": "web"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertIn({"name": "PORT", "value": "5000", "ptype": "web"}, response.data['values'])
+
+        # reset same config value
+        body = {'values': [{"name": "PORT", "value": "5001", "ptype": "web"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertIn({"name": "PORT", "value": "5001", "ptype": "web"}, response.data['values'])
+        # unset PORT
+        body = {'values': [{"name": "PORT", "value": None, "ptype": "web"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.data['values'], [], response.data)
+
+    def test_config_version(self, mock_requests):
+        """
+        Test that config sets on the same key function properly
+        """
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/config"
+
+        # set an initial config value
+        value1 = {"name": "PORT", "value": "5000", "group": "global"}
+        body = {'values': [value1]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([value1], response.data['values'])
+        url = f"/v2/apps/{app_id}/config"
+
+        # set config NAME
+        value2 = {"name": "NAME", "value": "drycc", "group": "global"}
+        body = {'values': [value2]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([value1, value2], response.data['values'])
+
+        # set config WEBSITE
+        value3 = {"name": "WEBSITE", "value": "www.drycc.cc", "group": "global"}
+        body = {'values': [value3]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual([value1, value2, value3], response.data['values'])
+
+        url = f"/v2/apps/{app_id}/config/?version=v2"
+        response = self.client.get(url)
+        self.assertEqual(response.data['values'], [value1], response.data)
+
+        url = f"/v2/apps/{app_id}/config/?version=v3"
+        response = self.client.get(url)
+        self.assertEqual(
+            response.data['values'], [value1, value2], response.data)
+
+        url = f"/v2/apps/{app_id}/config/?version=v4"
+        response = self.client.get(url)
+        self.assertEqual(
+            response.data['values'],
+            [value1, value2, value3], response.data)
+
+    def test_diff_ptypes(self, mock_requests):
+        app_id = self.create_app()
+        value1 = {"name": "WEBSITE", "value": "www.drycc.cc", "group": "global"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), {"web", "task"})
+
+        # add value
+        value1 = {"name": "DEBUG", "value": "true", "group": "mygroup"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), set())
+
+        # add group ref
+        body = {'values_refs': {"task": ["mygroup"]}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), {"task"})
+        # add new group
+        value1 = {"name": "DEBUG", "value": "true", "group": "mygroup1"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        body = {'values_refs': {"task": ["mygroup1"]}}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        uuid = response.data["uuid"]
+        config = Config.objects.get(uuid=uuid)
+        old_config = config.previous()
+        self.assertEqual(config.diff_ptypes(old_config, ["web", "task"]), {"task"})
+        self.assertEqual(config.envs("task"), {'WEBSITE': 'www.drycc.cc', 'DEBUG': 'true'})
+
+    def test_config_from_dryccfile(self, mock_requests):
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        build_body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'dryccfile': {
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "deploy": {
+                            'image': "127.0.0.1/cat/cat"
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "deploy": {
+                            'image': "127.0.0.1/cat/cat"
+                        },
+                    },
+                },
+            },
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+        value1 = {"name": "WEBSITE", "value": "www.drycc.cc", "group": "global"}
+        body = {'values': [value1]}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(release.config.envs("web"), {"WEBSITE": "www.drycc.cc"})
+        # set env by dryccfile
+        build_body['dryccfile']['config'] = {
+            "mygroup1": {
+                "GROUP": "g1",
+                "DEBUG": "tr",
+            },
+            "mygroup2": {
+                "TEST1": "g1",
+                "TEST2": "tr",
+            },
+        }
+        build_body['dryccfile']['pipeline']['web.yaml']['env'] = {
+            "PENV1": "web",
+            "PENV2": "web",
+        }
+        build_body['dryccfile']['pipeline']['web.yaml']['config'] = [
+            "mygroup1", "mygroup2"
+        ]
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(
+            release.config.envs("web"),
+            {
+                'GROUP': 'g1', 'DEBUG': 'tr', 'TEST1': 'g1',
+                'TEST2': 'tr', 'PENV1': 'web', 'PENV2': 'web'
+            },
+        )
+
+        # test use old config and healthcheck
+        new_build_body = copy.deepcopy(build_body)
+        del new_build_body['dryccfile']['config']
+        del new_build_body['dryccfile']['pipeline']['web.yaml']['env']
+        del new_build_body['dryccfile']['pipeline']['web.yaml']['config']
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, new_build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(release.config.values, [])
+        self.assertEqual(release.config.values_refs, {})
+        # set empty
+        new_build_body = copy.deepcopy(build_body)
+        new_build_body['dryccfile']['config'] = {}
+        new_build_body['dryccfile']['pipeline']['web.yaml']['env'] = {}
+        new_build_body['dryccfile']['pipeline']['web.yaml']['config'] = []
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, new_build_body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        self.assertEqual(release.config.envs("web"), {})
+        self.assertEqual(release.config.values_refs, {})
+
+        # test config modification creates new build with merged dryccfile
+        # First, create a build with dryccfile
+        build_body_with_dryccfile = copy.deepcopy(build_body)
+        build_body_with_dryccfile['dryccfile']['config'] = {
+            "mygroup1": {
+                "GROUP": "g1",
+                "DEBUG": "tr",
+            }
+        }
+        build_body_with_dryccfile['dryccfile']['pipeline']['web.yaml']['env'] = {
+            "PENV1": "web"
+        }
+        build_body_with_dryccfile['dryccfile']['pipeline']['web.yaml']['config'] = [
+            "mygroup1"
+        ]
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, build_body_with_dryccfile)
+            self.assertEqual(response.status_code, 201, response.data)
+
+        # Get the latest release and build
+        release = app.release_set.latest()
+        initial_build = release.build
+        self.assertIsNotNone(initial_build.dryccfile)
+
+        # Now modify config - this should trigger build.merge() and create a new build
+        config_values = [
+            {"name": "NEW_CONFIG", "value": "new_value", "group": "mygroup1"},
+            {"name": "PENV2", "value": "web2", "ptype": "web"}
+        ]
+        body = {'values': config_values}
+        url = f"/v2/apps/{app_id}/config"
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+        # Check that a new release was created
+        new_release = app.release_set.latest()
+        self.assertNotEqual(release.uuid, new_release.uuid)
+
+        # Check that a new build was created with merged dryccfile
+        new_build = new_release.build
+        self.assertIsNotNone(new_build)
+        self.assertNotEqual(initial_build.uuid, new_build.uuid)
+
+        # Verify dryccfile was merged with config values
+        self.assertIn('dryccfile', new_build.__dict__)
+        dryccfile = new_build.dryccfile
+
+        # Check config groups were merged
+        self.assertIn('config', dryccfile)
+        self.assertIn('mygroup1', dryccfile['config'])
+        self.assertEqual(dryccfile['config']['mygroup1']['GROUP'], 'g1')
+        self.assertEqual(dryccfile['config']['mygroup1']['DEBUG'], 'tr')
+        self.assertEqual(dryccfile['config']['mygroup1']['NEW_CONFIG'], 'new_value')
+
+        # Check ptype envs were merged
+        self.assertIn('pipeline', dryccfile)
+        self.assertIn('web.yaml', dryccfile['pipeline'])
+        self.assertIn('env', dryccfile['pipeline']['web.yaml'])
+        self.assertEqual(dryccfile['pipeline']['web.yaml']['env']['PENV1'], 'web')
+        self.assertEqual(dryccfile['pipeline']['web.yaml']['env']['PENV2'], 'web2')
+
+        # Check config refs
+        self.assertIn('config', dryccfile['pipeline']['web.yaml'])
+        self.assertEqual(dryccfile['pipeline']['web.yaml']['config'], ['mygroup1'])
+
+        # Verify config values are correct
+        config = new_release.config
+        expected_envs = {
+            'GROUP': 'g1',
+            'DEBUG': 'tr',
+            'NEW_CONFIG': 'new_value',
+            'PENV1': 'web',
+            'PENV2': 'web2'
+        }
+        self.assertEqual(config.envs("web"), expected_envs)
diff --git a/rootfs/api/tests/test_consumers.py b/rootfs/api/tests/test_consumers.py
new file mode 100644
index 000000000..15d84fe34
--- /dev/null
+++ b/rootfs/api/tests/test_consumers.py
@@ -0,0 +1,931 @@
+"""
+Unit tests for the Drycc api consumers.
+
+Run the tests with "./manage.py test api.tests.test_consumers"
+"""
+import json
+import asyncio
+import aiohttp
+from unittest import mock
+from unittest.mock import MagicMock, patch
+
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from channels.exceptions import DenyConnection
+
+from api.models.app import App
+from api.models.volume import Volume
+from api.consumers import (
+    AppPermChecker,
+    BaseAppConsumer,
+    BaseK8sConsumer,
+    AppPodLogsConsumer,
+    AppPodExecConsumer,
+    FilerProxyConsumer
+)
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class ConsumerTestCase(DryccTransactionTestCase):
+    """Base test case for consumer tests"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.app_id = self.create_app()
+        self.app = App.objects.get(id=self.app_id)
+
+    def tearDown(self):
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+
+class AppPermCheckerTest(ConsumerTestCase):
+    """Tests for AppPermChecker"""
+
+    def test_has_perm_with_logged_in_user(self):
+        """Test permission check with valid user and app"""
+        checker = AppPermChecker({
+            'user': self.user,
+            'url_route': {'kwargs': {'id': self.app_id}}
+        })
+
+        # Mock the async method
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertTrue(result[0])  # Should have permission
+            # When permission is granted, message is None
+
+        asyncio.run(test_async())
+
+    def test_has_perm_with_anonymous_user(self):
+        """Test permission check with anonymous user"""
+        checker = AppPermChecker({
+            'user': None,
+            'url_route': {'kwargs': {'id': self.app_id}}
+        })
+
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertFalse(result[0])  # Should not have permission
+            self.assertEqual(result[1], "user not login")
+
+        asyncio.run(test_async())
+
+    def test_has_perm_with_nonexistent_app(self):
+        """Test permission check with non-existent app"""
+        checker = AppPermChecker({
+            'user': self.user,
+            'url_route': {'kwargs': {'id': 'nonexistent-app'}}
+        })
+
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertFalse(result[0])  # Should not have permission
+            self.assertEqual(result[1], "app not exists")
+
+        asyncio.run(test_async())
+
+    def test_has_perm_with_cache(self):
+        """Test permission check with cached permissions"""
+        checker = AppPermChecker({
+            'user': self.user,
+            'url_route': {'kwargs': {'id': self.app_id}}
+        })
+
+        # Set cache
+        cache_key = f"permission:user:{self.user.id}:app:{self.app_id}"
+        cache.set(cache_key, (True, "cached permission"))
+
+        async def test_async():
+            result = await checker.has_perm()
+            self.assertTrue(result[0])
+            self.assertEqual(result[1], "cached permission")
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class BaseAppConsumerTest(ConsumerTestCase):
+    """Tests for BaseAppConsumer"""
+
+    def test_connect_with_permission(self, mock_requests):
+        """Test WebSocket connection with valid permissions"""
+        async def test_async():
+            # Mock the has_perm method to return True
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(BaseAppConsumer, 'accept') as mock_accept:
+                    consumer = BaseAppConsumer()
+                    consumer.scope = {
+                        'user': self.user,
+                        'url_route': {'kwargs': {'id': self.app_id}}
+                    }
+
+                    await consumer.connect()
+                    mock_accept.assert_called_once()
+
+        asyncio.run(test_async())
+
+    def test_connect_without_permission(self, mock_requests):
+        """Test WebSocket connection without valid permissions"""
+        async def test_async():
+            # Mock the has_perm method to return False
+            async def mock_has_perm():
+                return (False, "permission denied")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                consumer = BaseAppConsumer()
+                consumer.scope = {
+                    'user': self.user,
+                    'url_route': {'kwargs': {'id': self.app_id}}
+                }
+
+                with self.assertRaises(DenyConnection):
+                    await consumer.connect()
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class BaseK8sConsumerTest(ConsumerTestCase):
+    """Tests for BaseK8sConsumer"""
+
+    def test_kubernetes_property(self, mock_requests):
+        """Test kubernetes property initialization"""
+        consumer = BaseK8sConsumer()
+
+        # Mock the token file reading and Kubernetes API client
+        with patch('builtins.open', mock.mock_open(read_data='mock-token')):
+            with patch('api.consumers.Configuration') as mock_config:
+                with patch('api.consumers.core_v1_api.CoreV1Api') as mock_api:
+                    mock_config_instance = MagicMock()
+                    mock_config.return_value = mock_config_instance
+
+                    consumer.kubernetes
+
+                    # Verify configuration was called
+                    mock_config.assert_called_once()
+                    mock_api.assert_called_once()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class AppPodLogsConsumerTest(ConsumerTestCase):
+    """Tests for AppPodLogsConsumer"""
+
+    def test_connect(self, mock_requests):
+        """Test WebSocket connection for pod logs"""
+        async def test_async():
+            with patch.object(BaseAppConsumer, 'connect') as mock_super_connect:
+                consumer = AppPodLogsConsumer()
+                consumer.scope = {
+                    'user': self.user,
+                    'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+                }
+
+                await consumer.connect()
+
+                mock_super_connect.assert_called_once()
+                self.assertFalse(consumer.running)
+                self.assertIsNone(consumer.response)
+                self.assertTrue(consumer.conneted)
+                self.assertEqual(consumer.buffer, b'')
+                self.assertEqual(consumer.pod_name, 'test-pod')
+
+        asyncio.run(test_async())
+
+    def test_receive_log_request(self, mock_requests):
+        """Test receiving log request message"""
+        async def test_async():
+            consumer = AppPodLogsConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+            }
+            consumer.running = False
+            consumer.id = self.app_id
+            consumer.pod_name = 'test-pod'
+
+            # Mock kubernetes client
+            mock_k8s = MagicMock()
+            mock_response = MagicMock()
+            mock_k8s.read_namespaced_pod_log.return_value = mock_response
+
+            with patch.object(type(consumer), 'kubernetes', new_callable=lambda: mock_k8s):
+                # Create a proper async function instead of using AsyncMock
+                async def mock_sync_to_async_func(*args, **kwargs):
+                    return mock_response
+
+                with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                    mock_sync_to_async.return_value = mock_sync_to_async_func
+                    with patch.object(consumer, 'send'):
+                        with patch.object(consumer, 'close'):
+
+                            test_data = {
+                                "lines": 100,
+                                "follow": False,
+                                "previous": False,
+                                "container": "web"
+                            }
+
+                            await consumer.receive(text_data=json.dumps(test_data))
+
+                            self.assertTrue(consumer.running)
+
+        asyncio.run(test_async())
+
+    def test_disconnect(self, mock_requests):
+        """Test WebSocket disconnection"""
+        async def test_async():
+            consumer = AppPodLogsConsumer()
+            consumer.conneted = True
+            consumer.response = MagicMock()
+            consumer.response.connection.sock = MagicMock()
+
+            with patch('asyncio.get_event_loop') as mock_loop:
+                mock_event_loop = MagicMock()
+                mock_loop.return_value = mock_event_loop
+                with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_sync_close():
+                        pass
+                    mock_sync_to_async.return_value = mock_sync_close
+
+                    await consumer.disconnect(1000)
+
+                    self.assertFalse(consumer.conneted)
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class AppPodExecConsumerTest(ConsumerTestCase):
+    """Tests for AppPodExecConsumer"""
+
+    def test_connect(self, mock_requests):
+        """Test WebSocket connection for pod exec"""
+        async def test_async():
+            with patch.object(BaseAppConsumer, 'connect') as mock_super_connect:
+                consumer = AppPodExecConsumer()
+                consumer.scope = {
+                    'user': self.user,
+                    'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+                }
+                await consumer.connect()
+                mock_super_connect.assert_called_once()
+                self.assertIsNone(consumer.stream)
+                self.assertTrue(consumer.conneted)
+                self.assertEqual(consumer.pod_name, 'test-pod')
+
+        asyncio.run(test_async())
+
+    def test_send_stdout(self, mock_requests):
+        """Test sending stdout data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+
+            with patch.object(consumer.__class__.__bases__[0], 'send') as mock_super_send:
+                # Test sending bytes data
+                await consumer.send(b'test output', channel=1)  # STDOUT_CHANNEL
+                mock_super_send.assert_called_with(bytes_data=b'\x01test output')
+
+                # Test sending string data
+                await consumer.send('test string', channel=1)
+                mock_super_send.assert_called_with(text_data='\x01test string')
+
+        asyncio.run(test_async())
+
+    def test_send_none_data(self, mock_requests):
+        """Test sending None data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+
+            with patch.object(consumer.__class__.__bases__[0], 'send') as mock_super_send:
+                await consumer.send(None)
+                mock_super_send.assert_not_called()
+
+        asyncio.run(test_async())
+
+    def test_receive_exec_request(self, mock_requests):
+        """Test receiving exec request"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {'kwargs': {'name': 'test-pod', 'id': self.app_id}}
+            }
+            consumer.stream = None
+            consumer.pod_name = 'test-pod'
+            consumer.id = self.app_id
+
+            exec_request = {
+                "command": ["/bin/bash"],
+                "stdin": True,
+                "tty": True
+            }
+
+            # Mock the kubernetes property
+            mock_k8s = MagicMock()
+            mock_k8s.connect_get_namespaced_pod_exec = MagicMock()
+
+            with patch.object(type(consumer), 'kubernetes', new_callable=lambda: mock_k8s):
+                with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                    with patch('api.consumers.stream'):
+                        mock_stream_obj = MagicMock()
+                        # Create a proper async function instead of AsyncMock
+
+                        async def mock_sync_to_async_func(*args, **kwargs):
+                            return mock_stream_obj
+                        mock_sync_to_async.return_value = mock_sync_to_async_func
+
+                        # Mock the task method to avoid creating unawaited coroutine
+                        async def mock_task():
+                            pass
+                        with patch.object(
+                            consumer, 'task', side_effect=mock_task
+                        ) as mock_task_method:
+                            await consumer.receive(text_data=json.dumps(exec_request))
+                            self.assertEqual(consumer.stream, mock_stream_obj)
+                            mock_task_method.assert_called_once()
+
+        asyncio.run(test_async())
+
+    def test_receive_command_data(self, mock_requests):
+        """Test receiving command input data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            mock_stream = MagicMock()
+            consumer.stream = mock_stream
+
+            with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                # Create a proper async function instead of AsyncMock
+                async def mock_write_stdin(channel, data):
+                    pass
+                mock_sync_to_async.return_value = mock_write_stdin
+
+                # Test text data
+                await consumer.receive(text_data='\x00test input')
+                mock_sync_to_async.assert_called()
+
+        asyncio.run(test_async())
+
+    def test_receive_invalid_operation(self, mock_requests):
+        """Test receiving data when stream is None and no valid text data"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            consumer.stream = None
+
+            with self.assertRaises(ValueError) as context:
+                await consumer.receive(bytes_data=b'invalid')
+
+            self.assertEqual(str(context.exception), "This operation is not supported!")
+
+        asyncio.run(test_async())
+
+    def test_disconnect(self, mock_requests):
+        """Test WebSocket disconnection"""
+        async def test_async():
+            consumer = AppPodExecConsumer()
+            consumer.stream = MagicMock()
+            consumer.conneted = True
+
+            with patch('api.consumers.sync_to_async') as mock_sync_to_async:
+                # Create a proper async function instead of AsyncMock
+                async def mock_close_stream():
+                    pass
+                mock_sync_to_async.return_value = mock_close_stream
+
+                await consumer.disconnect(1000)
+
+                self.assertFalse(consumer.conneted)
+
+        asyncio.run(test_async())
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class FilerProxyConsumerTest(ConsumerTestCase):
+    """Tests for FilerProxyConsumer"""
+
+    def setUp(self):
+        super().setUp()
+        # Create a test volume using Volume.objects.create to avoid k8s calls
+        # We'll patch the save method to prevent k8s interactions
+        with patch.object(Volume, 'save_to_k8s'), patch.object(Volume, 'delete_from_k8s'):
+            self.test_volume = Volume.objects.create(
+                app=self.app,
+                name='test-volume',
+                size='1G',
+                path={'web': '/data'},
+                type='csi',
+                parameters={'csi': {'driver': 'test-driver'}}
+            )
+
+    def test_handle_without_permission(self, mock_requests):
+        """Test handling request without permission"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/_/ping',
+                'url_route': {
+                    'kwargs': {'path': '_/ping', 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            async def mock_has_perm():
+                return (False, "permission denied")
+
+            async def mock_bind_func():
+                return {'endpoint': '', 'username': 'test', 'password': 'test'}
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+                        await consumer.handle(b'')
+                        mock_send_response_obj.assert_awaited_once_with(403, b'permission denied')
+
+        asyncio.run(test_async())
+
+    def test_handle_ping_endpoint(self, mock_requests):
+        """Test handling ping endpoint"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/_/ping',
+                'url_route': {
+                    'kwargs': {'path': '_/ping', 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return {'username': 'test', 'password': 'test'}
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.info = mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'')
+
+                        mock_send_response_obj.assert_called_once_with(200, b"pong")
+
+        asyncio.run(test_async())
+
+    def test_handle_bind_endpoint(self, mock_requests):
+        """Test handling bind endpoint"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/',
+                'url_route': {
+                    'kwargs': {'path': '_/bind', 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            filer_data = {
+                'username': 'test-username',
+                'password': 'test-password'
+            }
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return filer_data
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'')
+
+                        expected_response = json.dumps({
+                            "username": "test-username",
+                            "password": "test-password"
+                        }).encode('utf-8')
+
+                        mock_send_response_obj.assert_called_once_with(200, expected_response)
+
+        asyncio.run(test_async())
+
+    def test_handle_proxy_request(self, mock_requests):
+        """Test handling proxy request to filer"""
+        path = 'webdav/files/test.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/test-volume/filer/{path}',
+                'url_route': {
+                    'kwargs': {'path': path, 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': [(b'host', b'example.com'), (b'user-agent', b'test-agent')],
+                'query_string': 'param=value'
+            }
+            # Set the id attribute that FilerProxyConsumer expects
+            consumer.id = self.app_id
+
+            filer_data = {
+                'endpoint': 'http://filer.example.com',
+                'username': 'test-username',
+                'password': 'test-password'
+            }
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(consumer, '_handle_proxy_request') as mock_handle_proxy:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return filer_data
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.info = mock_client.bind = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'test body')
+
+                        expected_url = "http://filer.example.com{}?param=value".format(
+                            consumer.scope['path']
+                        )
+                        expected_headers = {'user-agent': 'test-agent'}
+
+                        mock_handle_proxy.assert_called_once_with(
+                            expected_url,
+                            'GET',
+                            expected_headers,
+                            b'test body'
+                        )
+
+        asyncio.run(test_async())
+
+    def test_forward_response(self, mock_requests):
+        """Test forwarding response from filer"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.config = MagicMock()
+            consumer.config.chunk_size = 1024
+
+            # Mock aiohttp response
+            mock_response = MagicMock()
+            mock_response.status = 200
+            mock_response.headers = {'content-type': 'application/json', 'content-length': '100'}
+
+            # Create an async iterator for iter_chunked
+            class AsyncIterator:
+                def __init__(self, items):
+                    self.items = iter(items)
+
+                def __aiter__(self):
+                    return self
+
+                async def __anext__(self):
+                    try:
+                        return next(self.items)
+                    except StopIteration:
+                        raise StopAsyncIteration
+
+            # Mock the iter_chunked method to return our async iterator
+            mock_response.content.iter_chunked = MagicMock(
+                return_value=AsyncIterator([b'chunk1', b'chunk2', b''])
+            )
+
+            # Mock async methods properly
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            async def mock_send_body(*args, **kwargs):
+                pass
+
+            async def mock_send_headers(*args, **kwargs):
+                pass
+
+            with patch.object(
+                consumer, 'send_body', side_effect=mock_send_body
+            ) as mock_send_body_obj:
+                with patch.object(
+                    consumer, 'send_headers', side_effect=mock_send_headers
+                ) as mock_send_headers_obj:
+                    await consumer._forward_response(mock_response)
+                    # Check that response headers were sent
+                    expected_headers = [
+                        [b'content-type', b'application/json'],
+                        [b'content-length', b'100']
+                    ]
+                    mock_send_headers_obj.assert_called_once_with(
+                        status=200,
+                        headers=expected_headers
+                    )
+                    # Check that body chunks were sent
+                    self.assertEqual(mock_send_body_obj.call_count, 3)
+
+        asyncio.run(test_async())
+
+    def test_handle_proxy_request_connection_error(self, mock_requests):
+        """Test handling connection error during proxy request"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.config = MagicMock()
+            consumer.config.timeout = aiohttp.ClientTimeout(total=30)
+
+            # Mock the entire session context manager flow
+            mock_session_instance = MagicMock()
+
+            async def mock_aenter(self):
+                return mock_session_instance
+
+            async def mock_aexit(self, *args):
+                return None
+            mock_session_instance.__aenter__ = mock_aenter
+            mock_session_instance.__aexit__ = mock_aexit
+
+            # Mock the request context manager to raise an exception
+            mock_request_cm = MagicMock()
+
+            async def mock_request_aenter(self):
+                raise aiohttp.ClientError("Connection failed")
+
+            async def mock_request_aexit(self, *args):
+                return None
+            mock_request_cm.__aenter__ = mock_request_aenter
+            mock_request_cm.__aexit__ = mock_request_aexit
+            mock_session_instance.request = MagicMock(return_value=mock_request_cm)
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch('aiohttp.ClientSession', return_value=mock_session_instance):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ):
+                    await consumer._handle_proxy_request(
+                        'http://example.com/test',
+                        'GET',
+                        {},
+                        b'proxy service unavailable: Connection failed'
+                    )
+
+        asyncio.run(test_async())
+
+    def test_handle_proxy_request_timeout(self, mock_requests):
+        """Test handling timeout during proxy request"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.config = MagicMock()
+            consumer.config.timeout = aiohttp.ClientTimeout(total=30)
+
+            # Mock the entire session context manager flow
+            mock_session_instance = MagicMock()
+
+            async def mock_aenter(self):
+                return mock_session_instance
+
+            async def mock_aexit(self, *args):
+                return None
+            mock_session_instance.__aenter__ = mock_aenter
+            mock_session_instance.__aexit__ = mock_aexit
+
+            # Mock the request context manager to raise a timeout
+            mock_request_cm = MagicMock()
+
+            async def mock_request_aenter(self):
+                raise asyncio.TimeoutError()
+
+            async def mock_request_aexit(self, *args):
+                return None
+            mock_request_cm.__aenter__ = mock_request_aenter
+            mock_request_cm.__aexit__ = mock_request_aexit
+            mock_session_instance.request = MagicMock(return_value=mock_request_cm)
+
+            with patch('aiohttp.ClientSession', return_value=mock_session_instance):
+                with patch.object(consumer, 'send_response') as mock_send_response:
+                    await consumer._handle_proxy_request(
+                        'http://example.com/test',
+                        'GET',
+                        {},
+                        b''
+                    )
+
+                    mock_send_response.assert_called_once_with(
+                        504,
+                        b'proxy request to backend filer timeout'
+                    )
+
+        asyncio.run(test_async())
+
+    def test_skip_request_headers(self, mock_requests):
+        """Test that certain headers are skipped in proxy requests"""
+        consumer = FilerProxyConsumer()
+
+        # Verify that SKIP_REQUEST_HEADERS contains expected headers
+        expected_skip_headers = {
+            'host', 'connection', 'keep-alive', 'proxy-connection',
+            'te', 'trailers', 'upgrade'
+        }
+
+        self.assertEqual(consumer.SKIP_REQUEST_HEADERS, expected_skip_headers)
+
+    def test_skip_response_headers(self, mock_requests):
+        """Test that certain headers are skipped in proxy responses"""
+        consumer = FilerProxyConsumer()
+
+        # Verify that SKIP_RESPONSE_HEADERS contains expected headers
+        expected_skip_headers = {
+            'connection', 'keep-alive', 'proxy-authenticate',
+            'proxy-authorization', 'te', 'trailers',
+            'transfer-encoding', 'upgrade'
+        }
+
+        self.assertEqual(consumer.SKIP_RESPONSE_HEADERS, expected_skip_headers)
+
+    def test_get_client_with_existing_volume(self, mock_requests):
+        """Test _get_client method with existing volume"""
+        path = 'test/file.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {
+                    'kwargs': {'path': path, 'name': 'test-volume', 'id': self.app_id}
+                }
+            }
+            consumer.id = self.app_id
+
+            with patch('api.filer.FilerClient') as mock_filer_client:
+                mock_client = MagicMock()
+                mock_filer_client.return_value = mock_client
+                # Test the _get_client method
+                client = await consumer._get_client(path)
+                # Verify that FilerClient was called with correct parameters
+                mock_filer_client.assert_called_once_with(self.app_id, self.test_volume, path)
+                self.assertEqual(client, mock_client)
+
+        asyncio.run(test_async())
+
+    def test_get_client_with_nonexistent_volume(self, mock_requests):
+        """Test _get_client method with non-existent volume"""
+        path = 'test/file.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'url_route': {
+                    'kwargs': {
+                        'path': path, 'name': 'nonexistent-volume', 'id': self.app_id,
+                    }
+                }
+            }
+            consumer.id = self.app_id
+            # Test the _get_client method
+            client = await consumer._get_client(path)
+            # Should return None when volume doesn't exist
+            self.assertIsNone(client)
+
+        asyncio.run(test_async())
+
+    def test_handle_with_volume_not_found(self, mock_requests):
+        """Test handling request when volume is not found"""
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            path = 'test/file.txt'
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/nonexistent-volume/filer/{path}',
+                'url_route': {
+                    'kwargs': {
+                        'path': path, 'name': 'nonexistent-volume', 'id': self.app_id,
+                    }
+                },
+                'method': 'GET',
+                'headers': []
+            }
+            consumer.id = self.app_id
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            async def mock_send_response(*args, **kwargs):
+                pass
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(
+                    consumer, 'send_response', side_effect=mock_send_response
+                ) as mock_send_response_obj:
+                    await consumer.handle(b'test body')
+                    mock_send_response_obj.assert_called_once_with(
+                        status=404,
+                        body=b'app or volume not found'
+                    )
+
+        asyncio.run(test_async())
+
+    def test_handle_with_existing_volume(self, mock_requests):
+        """Test handling request with existing volume"""
+        path = 'webdav/test/file.txt'
+
+        async def test_async():
+            consumer = FilerProxyConsumer()
+            consumer.scope = {
+                'user': self.user,
+                'path': f'/v2/apps/{self.app_id}/volumes/nonexistent-volume/filer/{path}',
+                'url_route': {
+                    'kwargs': {'path': path, 'name': 'test-volume', 'id': self.app_id}
+                },
+                'method': 'GET',
+                'headers': [(b'host', b'example.com'), (b'user-agent', b'test-agent')],
+                'query_string': 'param=value'
+            }
+            consumer.id = self.app_id
+
+            filer_data = {
+                'endpoint': 'http://filer.example.com',
+                'username': 'test-username',
+                'password': 'test-password'
+            }
+
+            async def mock_has_perm():
+                return (True, "permission granted")
+
+            with patch.object(AppPermChecker, 'has_perm', side_effect=mock_has_perm):
+                with patch.object(consumer, '_handle_proxy_request') as mock_handle_proxy:
+                    # Create a proper async function instead of AsyncMock
+                    async def mock_bind_func():
+                        return filer_data
+
+                    with patch('api.filer.FilerClient') as mock_filer_client:
+                        mock_client = MagicMock()
+                        mock_client.bind = mock_client.info = mock_bind_func
+                        mock_filer_client.return_value = mock_client
+
+                        await consumer.handle(b'test body')
+
+                        expected_url = "http://filer.example.com{}?param=value".format(
+                            consumer.scope['path']
+                        )
+                        expected_headers = {'user-agent': 'test-agent'}
+
+                        mock_handle_proxy.assert_called_once_with(
+                            expected_url,
+                            'GET',
+                            expected_headers,
+                            b'test body'
+                        )
+
+        asyncio.run(test_async())
diff --git a/rootfs/api/tests/test_domain.py b/rootfs/api/tests/test_domain.py
index b0200d3ff..918eea36d 100644
--- a/rootfs/api/tests/test_domain.py
+++ b/rootfs/api/tests/test_domain.py
@@ -7,8 +7,8 @@
 
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
+from api.models.base import PTYPE_WEB
 from api.models.domain import Domain
 from api.tests import DryccTestCase
 
@@ -25,7 +25,7 @@ class DomainTest(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         self.app_id = self.create_app()
 
@@ -39,19 +39,19 @@ def test_response_data(self):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': 'test-domain.example.com'}
+            {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB}
         )
         self.assertEqual(response.status_code, 201, response.data)
 
         for key in response.data:
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'domain'])
+            self.assertIn(
+                key, ['uuid', 'created', 'updated', 'app', 'domain', 'ptype'])
 
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'domain': 'test-domain.example.com'
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, expected | response.data)
 
     def test_strip_dot(self):
         """Test that a dot on the right side of the domain gets stripped"""
@@ -60,7 +60,7 @@ def test_strip_dot(self):
         url = '/v2/apps/{app_id}/domains'.format(app_id=self.app_id)
 
         # Create
-        response = self.client.post(url, {'domain': domain})
+        response = self.client.post(url, {'domain': domain, 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 201, msg)
 
         # Fetch
@@ -97,7 +97,8 @@ def test_manage_idn_domain(self):
                 unicode_domain = idna.decode(ace_domain)
 
             # Create
-            response = self.client.post(url, {'domain': domain})
+            response = self.client.post(
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, msg)
 
             # Fetch
@@ -108,12 +109,14 @@ def test_manage_idn_domain(self):
 
             # Verify creation failure for same domain with different encoding
             if ace_domain != domain:
-                response = self.client.post(url, {'domain': ace_domain})
+                response = self.client.post(
+                    url, {'domain': ace_domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 400, msg)
 
             # Verify creation failure for same domain with different encoding
             if unicode_domain != domain:
-                response = self.client.post(url, {'domain': unicode_domain})
+                response = self.client.post(
+                    url, {'domain': unicode_domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 400, msg)
 
             # Delete
@@ -134,7 +137,8 @@ def test_manage_idn_domain(self):
             # Use different encoding for creating and deleting (ACE)
             if ace_domain != domain:
                 # Create
-                response = self.client.post(url, {'domain': domain})
+                response = self.client.post(
+                    url, {'domain': domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 201, msg)
 
                 # Fetch
@@ -161,7 +165,8 @@ def test_manage_idn_domain(self):
             # Use different encoding for creating and deleting (Unicode)
             if unicode_domain != domain:
                 # Create
-                response = self.client.post(url, {'domain': domain})
+                response = self.client.post(
+                    url, {'domain': domain, 'ptype': PTYPE_WEB})
                 self.assertEqual(response.status_code, 201, msg)
 
                 # Fetch
@@ -206,7 +211,8 @@ def test_manage_domain(self):
             msg = "failed on '{}'".format(domain)
 
             # Create
-            response = self.client.post(url, {'domain': domain})
+            response = self.client.post(
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, msg)
 
             # Fetch
@@ -245,7 +251,8 @@ def test_delete_domain_does_not_remove_latest(self):
             'django.paas-sandbox',
         ]
         for domain in test_domains:
-            response = self.client.post(url, {'domain': domain})
+            response = self.client.post(
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, response.data)
 
         url = '/v2/apps/{app_id}/domains/{domain}'.format(domain=test_domains[0],
@@ -263,7 +270,8 @@ def test_delete_domain_does_not_remove_others(self):
     def test_manage_domain_invalid_app(self):
         # Create domain
         url = '/v2/apps/{app_id}/domains'.format(app_id="this-app-does-not-exist")
-        response = self.client.post(url, {'domain': 'test-domain.example.com'})
+        response = self.client.post(
+            url, {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 404)
 
         # verify
@@ -285,7 +293,8 @@ def test_manage_domain_invalid_domain(self):
         ]
         for domain in test_domains:
             msg = "failed on \"{}\"".format(domain)
-            response = self.client.post(url, {'domain': domain})
+            response = self.client.post(
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 400, msg)
 
     def test_admin_can_add_domains_to_other_apps(self):
@@ -293,14 +302,15 @@ def test_admin_can_add_domains_to_other_apps(self):
         domains to it.
         """
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
 
         app_id = self.create_app()
 
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         url = '/v2/apps/{}/domains'.format(app_id)
-        response = self.client.post(url, {'domain': 'example.drycc.example.com'})
+        response = self.client.post(
+            url, {'domain': 'example.drycc.example.com', 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 201, response.data)
 
     def test_unauthorized_user_cannot_modify_domain(self):
@@ -313,11 +323,12 @@ def test_unauthorized_user_cannot_modify_domain(self):
         app_id = self.create_app()
 
         unauthorized_user = User.objects.get(username='autotest2')
-        unauthorized_token = Token.objects.get(user=unauthorized_user).key
+        unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
 
         url = '/v2/apps/{}/domains'.format(app_id)
-        response = self.client.post(url, {'domain': 'example.com'})
+        response = self.client.post(
+            url, {'domain': 'example.com', 'ptype': PTYPE_WEB})
         self.assertEqual(response.status_code, 403)
 
     def test_kubernetes_service_failure(self):
@@ -330,5 +341,6 @@ def test_kubernetes_service_failure(self):
         with mock.patch('scheduler.resources.service.Service.patch'):
             domain = 'foo.com'
             url = '/v2/apps/{}/domains'.format(app_id)
-            response = self.client.post(url, {'domain': domain})
+            response = self.client.post(
+                url, {'domain': domain, 'ptype': PTYPE_WEB})
             self.assertEqual(response.status_code, 201, response.data)
diff --git a/rootfs/api/tests/test_event.py b/rootfs/api/tests/test_event.py
new file mode 100644
index 000000000..2b3b0d717
--- /dev/null
+++ b/rootfs/api/tests/test_event.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for the Drycc api app.
+
+Run the tests with "./manage.py test api"
+"""
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+
+from api.models.app import App
+
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class EventTest(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.app = App.objects.get(id=self.create_app())
+
+    def tearDown(self):
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def build_deploy(self, app_id):
+        # post a new build with procfile
+        url = "/v2/apps/{app_id}/build".format(app_id=app_id)
+        body = {
+            'image': 'autotest/example',
+            'sha': 'a'*40,
+            'stack': 'heroku-18',
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+    def test_events(self, mock_requests):
+        """
+        Test that config is auto-created for a new app and that
+        config can be updated using a PATCH
+        """
+        app_id = self.create_app()
+        self.build_deploy(app_id)
+
+        # list events of pod
+        url = f"/v2/apps/{app_id}/pods"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+        pod_name = response.data['results'][0]["name"]
+        response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
+        url = f"/v2/apps/{app_id}/events"
+        response = self.client.get(url, {"pod_name": pod_name})
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # list events of deployment
+        url = f"/v2/apps/{app_id}/events"
+        response = self.client.get(url, {"ptype": f"{app_id}-web"})
+        self.assertEqual(response.status_code, 200, response.data)
diff --git a/rootfs/api/tests/test_gateway.py b/rootfs/api/tests/test_gateway.py
index 415b676e4..dda6e3119 100644
--- a/rootfs/api/tests/test_gateway.py
+++ b/rootfs/api/tests/test_gateway.py
@@ -2,14 +2,17 @@
 import json
 import string
 import random
-import jsonschema
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
+from api.models.build import Build
+from api.models.gateway import Route
+from api.models.release import Release
+from api.utils import validate_json
 from api.tests import TEST_ROOT, DryccTransactionTestCase
-from api.schemas.rules import SCHEMA as RULES_SCHEMA
+from api.serializers.schemas.rules import SCHEMA as RULES_SCHEMA
 
 User = get_user_model()
 
@@ -19,13 +22,46 @@ class BaseGatewayTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
+    def create_app_with_domain_and_deploy(self):
+        app_id = self.create_app()
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        # check default gateway route
+        response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 0, response.data)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 0, response.data)
+        # create a release so we can scale
+        app = App.objects.get(id=app_id)
+        build = Build.objects.create(app=app, image="qwerty")
+
+        # create an initial release
+        release = Release.objects.create(
+            version=2,
+            app=app,
+            config=app.config_set.latest(),
+            build=build
+        )
+        # deploy
+        release.deploy()
+        return app_id
+
+    def change_certs_auto(self, app_id, enabled):
+        data = {'certs_auto_enabled': enabled}
+        response = self.client.post(f'/v2/apps/{app_id}/tls', data)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.data.get('certs_auto_enabled'), enabled, response.data)
+
     def create_gateway(self, app_id, name, port, protocol):
         response = self.client.post(
             '/v2/apps/{}/gateways/'.format(app_id),
@@ -33,8 +69,29 @@ def create_gateway(self, app_id, name, port, protocol):
         )
         self.assertEqual(response.status_code, 201)
 
+    def create_gateway_name(self):
+        app_id = self.create_app()
+        name1 = 'gatway_'
+        response = self.client.post(
+            f'/v2/apps/{app_id}/gateways/',
+            {'name': name1, 'port': 80, 'protocol': 'HTTP'}
+        )
+        self.assertEqual(response.status_code, 400)
+        name2 = '-gatway'
+        response = self.client.post(
+            f'/v2/apps/{app_id}/gateways/',
+            {'name': name2, 'port': 80, 'protocol': 'HTTP'}
+        )
+        self.assertEqual(response.status_code, 400)
+        name3 = 'test.gatway'
+        response = self.client.post(
+            f'/v2/apps/{app_id}/gateways/',
+            {'name': name3, 'port': 80, 'protocol': 'HTTP'}
+        )
+        self.assertEqual(response.status_code, 400)
+
     def create_tls_domain(self, app_id):
-        cert_url = '/v2/certs'
+        cert_url = f'/v2/apps/{app_id}/certs'
         secret_name = ''.join(random.choice(string.ascii_lowercase) for _ in range(23))
         domain = 'autotest.example.com'
 
@@ -55,10 +112,9 @@ def create_tls_domain(self, app_id):
 
         response = self.client.post(
             '/v2/apps/{}/domains'.format(app_id),
-            {'domain': domain}
+            {'domain': domain, 'ptype': PTYPE_WEB}
         )
         self.assertEqual(response.status_code, 201)
-
         response = self.client.post(
             '{}/{}/domain/'.format(cert_url, secret_name),
             {'domain': domain}
@@ -79,17 +135,22 @@ def test_create_gateway(self):
 
     def test_add_listener(self):
         app_id = self.create_app()
-        self.create_gateway(app_id, 'bing-gateway', 8000, "HTTP")
+        gateway_name = 'bing-gateway'
+        self.create_gateway(app_id, gateway_name, 8000, "HTTP")
         response = self.client.post(
             '/v2/apps/{}/gateways/'.format(app_id),
-            {'name': 'bing-gateway', 'port': 443, 'protocol': "HTTP"}
+            {'name': gateway_name, 'port': 443, 'protocol': "HTTP"}
         )
         self.assertEqual(response.status_code, 201)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
-        results = [{
+        actual = None
+        for result in response.data["results"]:
+            if result["name"] == gateway_name:
+                actual = json.loads(json.dumps(result))
+                break
+        expect = {
             "app": app_id,
-            "owner": "autotest",
-            "name": "bing-gateway",
+            "name": gateway_name,
             "listeners": [
                 {
                     "name": "tcp-8000-%s" % 0,
@@ -102,64 +163,139 @@ def test_add_listener(self):
                     "port": 443,
                     "protocol": "HTTP",
                     "allowedRoutes": {"namespaces": {"from": "All"}}
-                }],
+                }
+            ],
             "addresses": [{
                 "type": "IPAddress",
                 "value": "172.22.108.207"
             }]
-        }]
-        self.assertEqual(results, json.loads(json.dumps(response.data["results"])))
+        }
+        self.assertEqual(expect, actual)
 
-    def add_tls_listener(self, name, protocol):
-        app_id = self.create_app()
-        domain, secret_name = self.create_tls_domain(app_id)
+    def add_listener(self, app_id, name, protocol, port):
         response = self.client.post(
             '/v2/apps/{}/gateways/'.format(app_id),
-            {'name': name, 'port': 443, 'protocol': protocol}
+            {'name': name, 'port': port, 'protocol': protocol}
         )
         self.assertEqual(response.status_code, 201)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
-        if protocol == "HTTPS":
-            listener_name = "mix-443-%s" % 0
-        else:
-            listener_name = "tcp-443-%s" % 0
-        results = [{
-            "app": app_id,
-            "owner": "autotest",
-            "name": name,
-            "listeners": [{
-                "tls": {
-                    "certificateRefs": [{
-                        "kind": "Secret",
-                        "name": secret_name
+        return app_id, response.data["results"]
+
+    def add_tls_listener(self, name, protocol, port):
+        app_id = self.create_app(name)
+        domain, secret_name = self.create_tls_domain(app_id)
+        app_id, results = self.add_listener(app_id, name, protocol, port)
+        return app_id, domain, secret_name, results
+
+    def test_add_tls_listener(self):
+        port = 443
+        name = "tls-gateway"
+        _, domain, secret_name, results = self.add_tls_listener(name, "TLS", port)
+        expect = [{
+            'app': name, 'name': name,
+            'listeners': [{
+                'allowedRoutes': {'namespaces': {'from': 'All'}},
+                'name': 'tcp-443-1', 'port': 443, 'hostname': domain,
+                'protocol': 'TLS',
+                'tls': {'certificateRefs': [
+                    {'kind': 'Secret', 'name': f"{secret_name}-certificate"}]}
+            }],
+            'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
+        }]
+        self.assertEqual(results, expect)
+
+    def test_add_https_listener(self):
+        port = 443
+        name = "bingo-gateway"
+        _, domain, secret_name, results = self.add_tls_listener(name, "HTTPS", port)
+        expect = [{
+            'app': name,
+            'name': name,
+            'listeners': [{
+                'allowedRoutes': {
+                    'namespaces': {
+                        'from': 'All'
+                    }
+                },
+                'name': 'mix-%s-1' % port,
+                'port': port,
+                'hostname': domain,
+                'protocol': 'HTTPS',
+                'tls': {
+                    'certificateRefs': [{
+                        'kind': 'Secret',
+                        'name': f"{secret_name}-certificate"
                     }]
+                }
+            }],
+            'addresses': [{
+                'type': 'IPAddress',
+                'value': '172.22.108.207'
+            }]
+        }]
+        self.assertEqual(results, expect)
+
+    def test_add_http_listener(self):
+        port = 80
+        name = "bingo-gateway"
+        _, domain, secret_name, results = self.add_tls_listener(name, "HTTP", port)
+        expect = [{
+            'app': 'bingo-gateway', 'name': 'bingo-gateway',
+            'listeners': [
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-1',
+                    'port': 80,
+                    'hostname': domain,
+                    'protocol': 'HTTP'
                 },
-                "name": listener_name,
-                "port": 443,
-                "hostname": domain,
-                "protocol": protocol,
-                "allowedRoutes": {
-                    "namespaces": {
-                        "from": "All"
-                    }
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-0',
+                    'port': 80,
+                    'protocol': 'HTTP'
+                },
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'mix-443-1',
+                    'port': 443,
+                    'hostname': domain,
+                    'protocol': 'HTTPS',
+                    'tls': {'certificateRefs': [
+                        {'kind': 'Secret', 'name': f"{secret_name}-certificate"}]}
                 }
+            ],
+            'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
+        }]
+        self.assertEqual(results, expect)
+
+    def test_add_udp_listener(self):
+        port = 999
+        name = "bingo-gateway"
+        app_id = self.create_app(name)
+        _, results = self.add_listener(app_id, name, "UDP", port)
+        expect = [{
+            'app': 'bingo-gateway',
+            'name': 'bingo-gateway',
+            'listeners': [{
+                'allowedRoutes': {
+                    'namespaces': {
+                        'from': 'All'
+                    }
+                },
+                'name': 'udp-999-0',
+                'port': 999,
+                'protocol': 'UDP'
             }],
-            "addresses": [{
-                "type": "IPAddress",
-                "value": "172.22.108.207"
+            'addresses': [{
+                'type': 'IPAddress',
+                'value': '172.22.108.207'
             }]
         }]
-        self.assertEqual(results, json.loads(json.dumps(response.data["results"])))
-        return app_id, domain, secret_name
-
-    def test_add_tls_listener(self):
-        self.add_tls_listener("tls-gateway", "TLS")
-
-    def test_add_https_listener(self):
-        self.add_tls_listener("bingo-gateway", "HTTPS")
+        self.assertEqual(results, expect)
 
     def test_remove_domain(self):
-        app_id, domain, _ = self.add_tls_listener("bingo-gateway", "HTTPS")
+        app_id, domain, _, _ = self.add_tls_listener("bingo-gateway", "HTTPS", 443)
         url = '/v2/apps/{app_id}/domains/{domain}'.format(domain=domain,
                                                           app_id=app_id)
         response = self.client.delete(url)
@@ -169,14 +305,13 @@ def test_remove_domain(self):
         self.assertEqual(response.data["results"][0]["listeners"], [])
 
     def test_remove_tls(self):
-        app_id, domain, secret_name = self.add_tls_listener("bingo-gateway", "HTTPS")
+        app_id, domain, secret_name, _ = self.add_tls_listener("bingo-gateway", "HTTPS", 443)
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
         self.assertEqual(len(response.data["results"][0]["listeners"]), 1)
         response = self.client.delete(
-            '{}/{}/domain/{}/'.format('/v2/certs', secret_name, domain)
+            '{}/{}/domain/{}/'.format(f'/v2/apps/{app_id}/certs', secret_name, domain)
         )
         self.assertEqual(response.status_code, 204)
-
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
         self.assertEqual(response.data["results"][0]["listeners"], [])
         return app_id
@@ -205,36 +340,125 @@ def test_remove_listener(self):
         response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
         self.assertEqual(response.data["count"], 0, response.data)
 
+    def test_change_tls(self):
+        app_id = self.create_app_with_domain_and_deploy()
+        response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
+        expect = [{
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
+                }
+            },
+            "name": "tcp-80-1",
+            "port": 80,
+            "hostname": "test-domain.example.com",
+            "protocol": "HTTP"
+        }, {
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
+                }
+            },
+            "name": "tcp-80-0",
+            "port": 80,
+            "protocol": "HTTP"
+        }]
+        self.assertEqual(response.data["count"], 1, response.data)
+        self.assertEqual(response.data["results"][0]["listeners"], expect, response.data)
+
+        self.change_certs_auto(app_id, True)
+        response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
+        self.assertEqual(response.data["count"], 1, response.data)
+        expect = [{
+            'app': app_id,
+            'name': app_id,
+            'listeners': [
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-1',
+                    'port': 80,
+                    'hostname': 'test-domain.example.com',
+                    'protocol': 'HTTP'
+                },
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'tcp-80-0',
+                    'port': 80,
+                    'protocol': 'HTTP'
+                },
+                {
+                    'allowedRoutes': {'namespaces': {'from': 'All'}},
+                    'name': 'mix-443-1',
+                    'port': 443,
+                    'hostname': 'test-domain.example.com',
+                    'protocol': 'HTTPS',
+                    'tls': {'certificateRefs': [{'kind': 'Secret', 'name': f'{app_id}-auto-tls'}]}
+                }
+            ],
+            'addresses': [{'type': 'IPAddress', 'value': '172.22.108.207'}]
+        }]
+        self.assertEqual(response.data["results"], expect, response.data)
+
+        self.change_certs_auto(app_id, False)
+        response = self.client.get('/v2/apps/{}/gateways/'.format(app_id))
+        expect = [{
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
+                }
+            },
+            "name": "tcp-80-1",
+            "port": 80,
+            "hostname": "test-domain.example.com",
+            "protocol": "HTTP"
+        }, {
+            "allowedRoutes": {
+                "namespaces": {
+                    "from": "All"
+                }
+            },
+            "name": "tcp-80-0",
+            "port": 80,
+            "protocol": "HTTP"
+        }]
+        self.assertEqual(response.data["count"], 1, response.data)
+        self.assertEqual(response.json()["results"][0]["listeners"], expect, response.data)
+
 
 class RouteTest(BaseGatewayTest):
 
-    def create_route(self, app_id):
+    def create_route(self, app_id, route_name="test-route",
+                     ptype="task", kind="HTTPRoute"):
         # create service
         port = 5000
-        procfile_type = "task"
         response = self.client.post(
             '/v2/apps/{}/services'.format(app_id),
             {
                 'port': port,
                 'protocol': 'TCP',
                 'target_port': port,
-                'procfile_type': procfile_type
+                'ptype': ptype
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
         # create route
-        route_name = "test-route"
         response = self.client.post(
             '/v2/apps/{}/routes/'.format(app_id),
             {
-                "port": 5000,
-                "procfile_type": procfile_type,
-                "kind": "HTTPRoute",
+                "kind": kind,
                 "name": route_name,
+                "rules": [{
+                    "backendRefs": [{
+                        "kind": "Service",
+                        "name": f"{app_id}-{ptype}",
+                        "port": 5000,
+                        "weight": 100,
+                    }],
+                }],
             }
         )
-        self.assertEqual(response.status_code, 201)
-        return procfile_type, port, route_name
+        self.assertEqual(response.status_code, 201, response.data)
+        return ptype, port, route_name
 
     def test_create_route(self):
         app_id = self.create_app()
@@ -244,16 +468,82 @@ def test_create_route(self):
             '/v2/apps/{}/routes/'.format(app_id),
             {
                 "port": 5000,
-                "procfile_type": "no-exists",
+                "ptype": "no-exists",
                 "kind": "HTTPRoute",
                 "name": "test-route-1",
             }
         )
         self.assertEqual(response.status_code, 404)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(response.data["count"], 1)
+        self.assertEqual(len(response.data["results"][0]["rules"]), 1)
+
+        # name regex format
+        name1 = "test-route_"
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "port": 5000,
+                "ptype": "no-exists",
+                "kind": "HTTPRoute",
+                "name": name1,
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+        # name regex format
+        name2 = "-test-route"
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "port": 5000,
+                "ptype": "no-exists",
+                "kind": "HTTPRoute",
+                "name": name2,
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+        name3 = "test.route"
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "port": 5000,
+                "ptype": "no-exists",
+                "kind": "HTTPRoute",
+                "name": name3,
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+
+    def test_hostnames(self):
+        app_id = self.create_app()
+        ptype, port, route_name = self.create_route(app_id)
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': "domain1.test.com", 'ptype': ptype}
+        )
+        self.assertEqual(response.status_code, 201)
+        route = Route.objects.get(app__id=app_id)
+        self.assertEqual(route.hostnames, ["domain1.test.com"])
+
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': "domain2.test.com", 'ptype': "web-1"}
+        )
+        self.assertEqual(response.status_code, 201)
+        route = Route.objects.get(app__id=app_id)
+        self.assertEqual(route.hostnames, ["domain1.test.com"])
+
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': "domain3.test.com", 'ptype': ptype}
+        )
+        self.assertEqual(response.status_code, 201)
+        route = Route.objects.get(app__id=app_id)
+        self.assertEqual(route.hostnames, ["domain1.test.com", "domain3.test.com"])
 
     def test_route_attach(self):
         app_id = self.create_app()
-        procfile_type, port, route_name = self.create_route(app_id)
+        ptype, port, route_name = self.create_route(app_id)
         gateway_name_1 = 'bing-gateway-1'
         self.create_gateway(app_id, gateway_name_1, 5000, "HTTP")
         self.client.patch(
@@ -276,7 +566,59 @@ def test_route_attach(self):
         )
         response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
         self.assertEqual(len(response.data["results"][0]["parent_refs"]), 2)
-        return procfile_type, app_id, gateway_name_1, gateway_name_2, port, route_name
+        return ptype, app_id, gateway_name_1, gateway_name_2, port, route_name
+
+    def test_route_check_parent_ok(self):
+        app_id = self.create_app()
+        _, port, route_name = self.create_route(app_id, "myroute1", "test")
+        gateway_name_1 = 'bing-gateway-1'
+        self.create_gateway(app_id, gateway_name_1, 5000, "HTTP")
+        response = self.client.patch(
+            '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
+            {
+                "gateway": gateway_name_1,
+                "port": port
+            }
+        )
+        self.assertEqual(response.status_code, 204, response.data)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"][0]["parent_refs"]), 1)
+        # create other route
+        _, port, route_name = self.create_route(app_id, "myroute2", "mytest")
+        response = self.client.patch(
+            '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
+            {
+                "gateway": gateway_name_1,
+                "port": port
+            }
+        )
+        self.assertEqual(response.status_code, 204)
+
+    def test_route_check_parent_err(self):
+        app_id = self.create_app()
+        _, port, route_name = self.create_route(app_id, "myroute1", "test", "TCPRoute")
+        gateway_name_1 = 'bing-gateway-1'
+        self.create_gateway(app_id, gateway_name_1, 5000, "TCP")
+        self.client.patch(
+            '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
+            {
+                "gateway": gateway_name_1,
+                "port": port
+            }
+        )
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"][0]["parent_refs"]), 1)
+        # create other route
+        _, port, route_name = self.create_route(app_id, "myroute2", "mytest", "TCPRoute")
+        response = self.client.patch(
+            '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
+            {
+                "gateway": gateway_name_1,
+                "port": port
+            }
+        )
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(response.data['detail'], 'this listener has already been referenced')
 
     def test_route_detach(self):
         _, app_id, gateway_name_1, gateway_name_2, port, route_name = self.test_route_attach()
@@ -318,14 +660,14 @@ def test_app_settings_change_routable(self):
         )
         self.assertEqual(response.status_code, 201, response.data)
         app = App.objects.get(id=app_id)
-        response = app._scheduler.httproute.get(app_id, route_name, ignore_exception=True)
+        response = app.scheduler.httproute.get(app_id, route_name, ignore_exception=True)
         self.assertEqual(response.status_code, 404)
         # Set routable to false
         response = self.client.post(
             f'/v2/apps/{app_id}/settings',
             {'routable': True}
         )
-        response = app._scheduler.httproute.get(app_id, route_name, ignore_exception=True)
+        response = app.scheduler.httproute.get(app_id, route_name, ignore_exception=True)
         self.assertEqual(response.status_code, 200)
 
     def test_route_delete(self):
@@ -342,71 +684,40 @@ def test_route_delete(self):
 
     def test_rule_get(self):
         app_id = self.create_app()
-        procfile_type, _, route_name = self.create_route(app_id)
+        ptype, _, route_name = self.create_route(app_id)
         response = self.client.get(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
         )
-        expect = {
-            'canary': [{
-                'backendRefs': [{
-                    'kind': 'Service',
-                    'name': '%s-%s' % (app_id, procfile_type),
-                    'port': 5000,
-                    'weight': 100
-                }, {
-                    'kind': 'Service',
-                    'name': '%s-%s-canary' % (app_id, procfile_type),
-                    'port': 5000,
-                    'weight': 0
-                }]
-            }],
-            'stable': [{
-                'backendRefs': [{
-                    'kind': 'Service',
-                    'name': '%s-%s' % (app_id, procfile_type),
-                    'port': 5000,
-                    'weight': 100
-                }]
+        expect = [{
+            'backendRefs': [{
+                'kind': 'Service',
+                'name': '%s-%s' % (app_id, ptype),
+                'port': 5000,
+                'weight': 100
             }]
-        }
-        self.assertEqual(response.data["stable"], expect["stable"])
-        self.assertEqual(response.data["canary"], expect["canary"])
+        }]
+        self.assertEqual(response.data, expect)
 
     def test_schemas(self):
         for rule in os.listdir("{}/rules/".format(TEST_ROOT)):
             with open("{}/rules/{}".format(TEST_ROOT, rule)) as f:
                 data = f.read()
                 try:
-                    jsonschema.validate(data, RULES_SCHEMA)
+                    validate_json(json.loads(data), RULES_SCHEMA)
                 except Exception as e:
                     raise self.failureException("validate %s rule error: %s" % (rule, str(e)))
 
     def test_rule_set(self):
         app_id = self.create_app()
-        procfile_type, _, route_name = self.create_route(app_id)
-        expect = {
-            "canary": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }, {
-                    "kind": "Service",
-                    "name": "%s-%s-canary" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 0
-                }]
-            }],
-            "stable": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }]
+        ptype, _, route_name = self.create_route(app_id)
+        expect = [{
+            "backendRefs": [{
+                "kind": "Service",
+                "name": "%s-%s" % (app_id, ptype),
+                "port": 5000,
+                "weight": 100
             }]
-        }
+        }]
         response = self.client.put(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
             json.dumps(expect),
@@ -416,32 +727,130 @@ def test_rule_set(self):
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
         )
         self.assertEqual(json.dumps(response.data), json.dumps(expect))
-        expect = {
-            "canary": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s-noexits" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }, {
-                    "kind": "Service",
-                    "name": "%s-%s-noexits-canary" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 0
-                }]
-            }],
-            "stable": [{
-                "backendRefs": [{
-                    "kind": "Service",
-                    "name": "%s-%s-noexits" % (app_id, procfile_type),
-                    "port": 5000,
-                    "weight": 100
-                }]
+        expect = [{
+            "backendRefs": [{
+                "kind": "Service",
+                "name": "%s-%s-noexits" % (app_id, ptype),
+                "port": 5000,
+                "weight": 100
             }]
-        }
+        }]
         response = self.client.put(
             '/v2/apps/{}/routes/{}/rules/'.format(app_id, route_name),
             json.dumps(expect),
             content_type="application/json",
         )
         self.assertEqual(response.status_code, 400)
+
+    def test_change_tls(self):
+        app_id = self.create_app_with_domain_and_deploy()
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }]
+        # enable tls
+        self.change_certs_auto(app_id, True)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }, {
+            'name': f'{app_id}',
+            'port': 443
+        }]
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+        # disable tls
+        self.change_certs_auto(app_id, False)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }]
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+
+    def test_route_parent_refs(self):
+        app_id = self.create_app_with_domain_and_deploy()
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }]
+        # enable tls
+        self.change_certs_auto(app_id, True)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+        response = self.client.get('/v2/apps/{}/routes/'.format(app_id))
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        expect = [{
+            'name': f'{app_id}',
+            'port': 80
+        }, {
+            'name': f'{app_id}',
+            'port': 443
+        }]
+        self.assertEqual(len(response.data["results"]), 1, response.data)
+        self.assertEqual(response.data["results"][0]["parent_refs"], expect, response.data)
+        # add new domain
+        ptype, kind, route_name, domain = (
+            'gateway', "HTTPRoute", "myroute-1", 'test-domain-gateway.example.com')
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': domain, 'ptype': ptype}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        # add new services
+        response = self.client.post(
+            '/v2/apps/{}/services'.format(app_id),
+            {
+                'port': 9999,
+                'protocol': 'TCP',
+                'target_port': 9999,
+                'ptype': 'gateway'
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        # add new router
+        response = self.client.post(
+            '/v2/apps/{}/routes/'.format(app_id),
+            {
+                "kind": kind,
+                "name": route_name,
+                "rules": [{
+                    "backendRefs": [{
+                        "kind": "Service",
+                        "name": f"{app_id}-{ptype}",
+                        "port": 9999,
+                        "weight": 100,
+                    }],
+                }],
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.client.patch(
+            '/v2/apps/{}/routes/{}/attach/'.format(app_id, route_name),
+            {
+                "gateway": app_id,
+                "port": 443
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        route = Route.objects.get(name=route_name)
+        response = route.scheduler.httproutes.get(app_id, route_name)
+        self.assertEqual(response.status_code, 200, response.json())
+        self.assertEqual(len(response.json()['spec']['parentRefs']), 1)
+        # get gateway
+        response = route.scheduler.gateways.get(app_id, app_id)
+        listeners = [
+            listener for listener in response.json()['spec']['listeners']
+            if listener['port'] == 443
+        ]
+        self.assertEqual(len(listeners) > 1,  True)
+        hostnames = [listener['hostname'] for listener in listeners]
+        self.assertEqual(domain in hostnames,  True)
diff --git a/rootfs/api/tests/test_healthchecks.py b/rootfs/api/tests/test_healthchecks.py
index 863c323ee..b4e9db8a5 100644
--- a/rootfs/api/tests/test_healthchecks.py
+++ b/rootfs/api/tests/test_healthchecks.py
@@ -3,9 +3,7 @@
 
 from django.core.cache import cache
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 
-from api.models.app import App
 from api.tests import adapter, DryccTransactionTestCase
 
 User = get_user_model()
@@ -19,57 +17,13 @@ class TestHealthchecks(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
-    def test_healthchecks_validations(self, mock_requests):
-        """
-        Test that healthchecks validations work
-        """
-        app_id = self.create_app()
-
-        # Set one of the values that require a numeric value to a string
-        response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'values': json.dumps({'HEALTHCHECK_INITIAL_DELAY': 'horse'})}
-        )
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # test URL - Path is the only allowed thing
-        # Try setting various things such as query param
-
-        # query param
-        response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'values': json.dumps({'HEALTHCHECK_URL': '/health?testing=0'})}
-        )
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # fragment
-        response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'values': json.dumps({'HEALTHCHECK_URL': '/health#db'})}
-        )
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # netloc
-        response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'values': json.dumps({'HEALTHCHECK_URL': 'http://someurl.com/health/'})}
-        )
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # no path
-        response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'values': json.dumps({'HEALTHCHECK_URL': 'http://someurl.com'})}
-        )
-        self.assertEqual(response.status_code, 400, response.data)
-
     def test_config_healthchecks(self, mock_requests):
         """
         Test that healthchecks can be applied
@@ -79,7 +33,7 @@ def test_config_healthchecks(self, mock_requests):
             'healthcheck': {'web': {'readinessProbe': {'httpGet': {'port': 5000}}}}
         }
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             readiness_probe)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertIn('readinessProbe', response.data['healthcheck']['web'])
@@ -89,7 +43,7 @@ def test_config_healthchecks(self, mock_requests):
                                           {'httpGet': {'port': 5000},
                                            'successThreshold': 1}}}}
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             liveness_probe)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertIn('livenessProbe', response.data['healthcheck']['web'])
@@ -104,13 +58,13 @@ def test_config_healthchecks(self, mock_requests):
 
         # check that config fails if trying to unset non-existing healthcheck
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': {'invalid_proctype': None}})
-        self.assertEqual(response.status_code, 422, response.data)
+        self.assertEqual(response.status_code, 400, response.data)
 
         # remove a probeType
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': {'web': {'livenessProbe': None}}})
         self.assertEqual(response.status_code, 201, response.data)
         self.assertNotIn('livenessProbe', response.data['healthcheck']['web'])
@@ -118,26 +72,26 @@ def test_config_healthchecks(self, mock_requests):
 
         # check that config fails if trying to unset non-existing probeType
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': {'web': {'livenessProbe': None}}})
         self.assertEqual(response.status_code, 422, response.data)
 
         # check that config fails if trying to unset non-existing probeType
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': {'invalid_proctype': {'livenessProbe': None}}})
-        self.assertEqual(response.status_code, 422, response.data)
+        self.assertEqual(response.status_code, 400, response.data)
 
         # check that config fails if trying to unset non-existing probeType
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': {'web': None}})
         self.assertEqual(response.status_code, 201, response.data)
         self.assertNotIn('web', response.data['healthcheck'])
 
         # post a new build
         response = self.client.post(
-            "/v2/apps/{app_id}/builds".format(**locals()),
+            f"/v2/apps/{app_id}/build",
             {'image': 'quay.io/autotest/example', 'stack': 'container'}
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -150,7 +104,7 @@ def test_config_healthchecks_validations(self, mock_requests):
 
         # Set a probe different from liveness/readiness
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': json.dumps({'web': {'testProbe':
                                         {'httpGet': {'port': '50'}, 'initialDelaySeconds': "1"}}})}
         )
@@ -158,7 +112,7 @@ def test_config_healthchecks_validations(self, mock_requests):
 
         # Set one of the values that require a numeric value to a string
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': json.dumps({'web': {'livenessProbe':
                                         {'httpGet': {'port': '50'}, 'initialDelaySeconds': "t"}}})}
         )
@@ -166,90 +120,8 @@ def test_config_healthchecks_validations(self, mock_requests):
 
         # Don't set one of the mandatory value
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             {'healthcheck': json.dumps({'web': {'livenessProbe':
                                         {'httpGet': {'path': '/'}, 'initialDelaySeconds': 1}}})}
         )
         self.assertEqual(response.status_code, 400, response.data)
-
-        # set liveness success threshold to a non-1 value
-        # Don't set one of the mandatory value
-        response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'healthcheck': {'web': {'livenessProbe':
-                             {'httpGet': {'path': '/', 'port': 5000},
-                              'successThreshold': 5}}}}
-        )
-        self.assertEqual(response.status_code, 400, response.data)
-
-    def test_config_healthchecks_legacy(self, mock_requests):
-        """
-        Test that when a user uses `drycc config:set HEALTHCHECK_URL=/`, the config
-        object is rolled over to the `healthcheck` field.
-        """
-        app_id = self.create_app()
-        app = App.objects.get(id=app_id)
-
-        # Set healthcheck URL to get defaults set
-        response = self.client.post(
-            '/v2/apps/{app.id}/config'.format(**locals()),
-            {'values': json.dumps({'HEALTHCHECK_URL': '/health'})}
-        )
-        self.assertEqual(response.status_code, 201, response.data)
-        # this gets migrated to the new healtcheck format
-        self.assertNotIn('HEALTHCHECK_URL', response.data['values'])
-        # legacy defaults
-        expected = {'web': {
-            'livenessProbe': {
-                'initialDelaySeconds': 50,
-                'timeoutSeconds': 50,
-                'periodSeconds': 10,
-                'successThreshold': 1,
-                'failureThreshold': 3,
-                'httpGet': {
-                    'path': '/health'
-                }
-            },
-            'readinessProbe': {
-                'initialDelaySeconds': 50,
-                'timeoutSeconds': 50,
-                'periodSeconds': 10,
-                'successThreshold': 1,
-                'failureThreshold': 3,
-                'httpGet': {
-                    'path': '/health'
-                }
-            }
-            }
-        }
-        actual = app.config_set.latest().healthcheck
-        self.assertEqual(actual, expected)
-        # Now set all the envvars and check to make sure they are written properly
-        response = self.client.post(
-            '/v2/apps/{app.id}/config'.format(**locals()),
-            {
-                'values': json.dumps({
-                    'HEALTHCHECK_URL': '/health',
-                    'HEALTHCHECK_INITIAL_DELAY': '25',
-                    'HEALTHCHECK_TIMEOUT': '10',
-                    'HEALTHCHECK_PERIOD_SECONDS': '5',
-                    'HEALTHCHECK_SUCCESS_THRESHOLD': '2',
-                    'HEALTHCHECK_FAILURE_THRESHOLD': '2'})
-            }
-        )
-        self.assertEqual(response.status_code, 201, response.data)
-        # this gets migrated to the new healtcheck format
-        self.assertNotIn('HEALTHCHECK_INITIAL_DELAY', response.data['values'])
-        expected['web']['livenessProbe'] = {
-            'initialDelaySeconds': 25,
-            'timeoutSeconds': 10,
-            'periodSeconds': 5,
-            'successThreshold': 2,
-            'failureThreshold': 2,
-            'httpGet': {
-                'path': '/health'
-            }
-        }
-        expected['web']['readinessProbe'] = expected['web']['livenessProbe']
-        actual = app.config_set.latest().healthcheck
-        self.assertEqual(expected, actual)
diff --git a/rootfs/api/tests/test_hooks.py b/rootfs/api/tests/test_hooks.py
index 8ce39fe08..970ec5b5d 100644
--- a/rootfs/api/tests/test_hooks.py
+++ b/rootfs/api/tests/test_hooks.py
@@ -6,7 +6,6 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
@@ -57,7 +56,7 @@ class HookTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -70,12 +69,6 @@ def test_key_hook(self, mock_requests):
         # Create app to use
         app_id = self.create_app()
 
-        # give user permission to app
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': str(self.user)}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-
         # Create rsa key
         body = {'id': str(self.user), 'public': RSA_PUBKEY}
         response = self.client.post('/v2/keys', body)
@@ -107,12 +100,12 @@ def test_key_hook(self, mock_requests):
 
         # Make sure 404 is returned for a random app
         url = '/v2/hooks/keys/doesnotexist'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
         # Test app that exists
         url = '/v2/hooks/keys/{}'.format(app_id)
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data, {"autotest": [
             {'key': rsa_pub, 'fingerprint': '54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:f5'},
@@ -123,12 +116,12 @@ def test_key_hook(self, mock_requests):
 
         # Test against an app that exist but user does not
         url = '/v2/hooks/keys/{}/foooooo'.format(app_id)
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
         # Test against an app that exists and user that does
         url = '/v2/hooks/keys/{}/{}'.format(app_id, str(self.user))
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data, {"autotest": [
             {'key': rsa_pub, 'fingerprint': '54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:f5'},
@@ -140,7 +133,7 @@ def test_key_hook(self, mock_requests):
 
         # Fetch a valid ssh key
         url = '/v2/hooks/key/54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:f5'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data, {
             "username": str(self.user),
@@ -151,31 +144,29 @@ def test_key_hook(self, mock_requests):
 
         # Fetch an non-existent base64 encoded ssh key
         url = '/v2/hooks/key/54:6d:da:1f:91:b5:2b:6f:a2:83:90:c4:f9:73:76:wooooo'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
         # Fetch an invalid (not encoded) ssh key
         url = '/v2/hooks/key/nope'
-        response = self.client.get(url, HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+        response = self.client.get(url, HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 404)
 
     def test_build_hook(self, mock_requests):
         """Test creating a Build via an API Hook"""
         app_id = self.create_app()
-
-        build = {'username': 'autotest', 'app': app_id}
-        url = '/v2/hooks/build'.format(**locals())
+        url = '/v2/hooks/build'
         body = {'receive_user': 'autotest',
                 'receive_repo': app_id,
-                'image': '{app_id}:v2'.format(**locals()),
+                'image': f'{app_id}:v2',
                 'stack': 'container'}
         # post the build without an auth token
         self.client.credentials()
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 401, response.data)
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
@@ -183,8 +174,7 @@ def test_build_hook(self, mock_requests):
     def test_build_hook_slug_url(self, mock_requests):
         """Test creating a slug_url build via an API Hook"""
         app_id = self.create_app()
-        build = {'username': 'autotest', 'app': app_id}
-        url = '/v2/hooks/build'.format(**locals())
+        url = '/v2/hooks/build'
         body = {'receive_user': 'autotest',
                 'receive_repo': app_id,
                 'image': 'http://example.com/slugs/foo-12345354.tar.gz',
@@ -195,9 +185,9 @@ def test_build_hook_slug_url(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 401, response.data)
 
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
@@ -207,42 +197,36 @@ def test_build_hook_procfile(self, mock_requests):
         app_id = self.create_app()
 
         build = {'username': 'autotest', 'app': app_id}
-        url = '/v2/hooks/build'.format(**locals())
+        url = '/v2/hooks/build'
         PROCFILE = {'web': 'node server.js', 'worker': 'node worker.js'}
         SHA = 'ecdff91c57a0b9ab82e89634df87e293d259a3aa'
         body = {'receive_user': 'autotest',
                 'receive_repo': app_id,
-                'image': '{app_id}:v2'.format(**locals()),
+                'image': f'{app_id}:v2',
                 'stack': 'heroku-18',
                 'sha': SHA,
                 'procfile': PROCFILE}
 
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
 
         # make sure build fields were populated
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertIn('results', response.data)
-        build = response.data['results'][0]
+        build = response.data
         self.assertEqual(build['sha'], SHA)
         self.assertEqual(build['procfile'], PROCFILE)
 
         # test listing/retrieving container info
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'web', 'v2')
 
         # post the build without an auth token
         self.client.credentials()
@@ -253,41 +237,35 @@ def test_build_hook_dockerfile(self, mock_requests):
         """Test creating a Dockerfile build via an API Hook"""
         app_id = self.create_app()
         build = {'username': 'autotest', 'app': app_id}
-        url = '/v2/hooks/build'.format(**locals())
+        url = '/v2/hooks/build'
         SHA = 'ecdff91c57a0b9ab82e89634df87e293d259a3aa'
         DOCKERFILE = """FROM busybox
         CMD /bin/true"""
 
         body = {'receive_user': 'autotest',
                 'receive_repo': app_id,
-                'image': '{app_id}:v2'.format(**locals()),
+                'image': f'{app_id}:v2',
                 'stack': 'container',
                 'sha': SHA,
                 'dockerfile': DOCKERFILE}
-        # post the build with the builder auth key
+        # post the build with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('release', response.data)
         self.assertIn('version', response.data['release'])
         # make sure build fields were populated
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertIn('results', response.data)
-        build = response.data['results'][0]
+        build = response.data
         self.assertEqual(build['sha'], SHA)
         self.assertEqual(build['dockerfile'], DOCKERFILE)
         # test default container
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}/pods/"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        container = response.data['results'][0]
-        self.assertEqual(container['type'], 'web')
-        self.assertEqual(container['release'], 'v2')
-        # pod name is auto generated so use regex
-        self.assertRegex(container['name'], app_id + '-web-[0-9]{1,10}-[a-z0-9]{5}')
+        self.assertPodContains(response.data['results'], app_id, 'web', 'v2')
 
         # post the build without an auth token
         self.client.credentials()
@@ -297,23 +275,22 @@ def test_build_hook_dockerfile(self, mock_requests):
     def test_config_hook(self, mock_requests):
         """Test reading Config via an API Hook"""
         app_id = self.create_app()
-        url = '/v2/apps/{app_id}/config'.format(**locals())
+        url = f'/v2/apps/{app_id}/config'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('values', response.data)
         values = response.data['values']
         # prepare the config hook
-        config = {'username': 'autotest', 'app': app_id}
-        url = '/v2/hooks/config'.format(**locals())
+        url = '/v2/hooks/config'
         body = {'receive_user': 'autotest',
                 'receive_repo': app_id}
         # post without an auth token
         self.client.credentials()
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 401, response.data)
-        # post with the builder auth key
+        # post with the service key
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('values', response.data)
         self.assertEqual(values, response.data['values'])
@@ -323,7 +300,7 @@ def test_admin_can_hook(self, mock_requests):
         """
         """Test creating a Push via the API"""
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
 
         app_id = self.create_app()
@@ -334,12 +311,12 @@ def test_admin_can_hook(self, mock_requests):
         """
         body = {'receive_user': 'autotest',
                 'receive_repo': app_id,
-                'image': '{app_id}:v2'.format(**locals()),
+                'image': f'{app_id}:v2',
                 'stack': 'container',
                 'sha': 'ecdff91c57a0b9ab82e89634df87e293d259a3aa',
                 'dockerfile': DOCKERFILE}
         url = '/v2/hooks/build'
         response = self.client.post(url, body,
-                                    HTTP_X_DRYCC_BUILDER_AUTH=settings.BUILDER_KEY)
+                                    HTTP_X_DRYCC_SERVICE_KEY=settings.SERVICE_KEY)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(response.data['release']['version'], 2)
diff --git a/rootfs/api/tests/test_key.py b/rootfs/api/tests/test_key.py
index 912ee1ceb..7fc377fa7 100644
--- a/rootfs/api/tests/test_key.py
+++ b/rootfs/api/tests/test_key.py
@@ -6,7 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from rest_framework.authtoken.models import Token
 
 from api.models.key import Key
 from api.utils import fingerprint
@@ -64,7 +63,7 @@ class KeyTest(DryccTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -85,7 +84,7 @@ def _check_key(self, pubkey):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 1)
 
-        url = '/v2/keys/{key_id}'.format(**locals())
+        url = f'/v2/keys/{key_id}'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(body['id'], response.data['id'])
@@ -178,10 +177,10 @@ def test_rsa_key_fingerprint(self):
 
     def test_key_api_with_non_superuser_rsa(self):
         self.user = User.objects.get(username='autotest2')
-        self.token = self.user.auth_token.key
+        self.token = self.get_or_create_token(self.user)
         self._check_key(RSA_PUBKEY)
 
     def test_key_api_with_non_superuser_ecdsa(self):
         self.user = User.objects.get(username='autotest2')
-        self.token = self.user.auth_token.key
+        self.token = self.get_or_create_token(self.user)
         self._check_key(ECDSA_PUBKEY)
diff --git a/rootfs/api/tests/test_limit.py b/rootfs/api/tests/test_limit.py
new file mode 100644
index 000000000..a3102e9d6
--- /dev/null
+++ b/rootfs/api/tests/test_limit.py
@@ -0,0 +1,106 @@
+import json
+import requests_mock
+
+from django.core.cache import cache
+from django.contrib.auth import get_user_model
+
+from api.tests import adapter, DryccTransactionTestCase
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class TestLimit(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+    def tearDown(self):
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def test_get_limits_specs(self, mock_requests):
+        results = {
+            "id": "std1",
+            "cpu": {
+                "name": "Universal CPU",
+                "boost": "3700MHZ",
+                "clock": "3100MHZ",
+                "cores": 32,
+                "threads": 64
+            },
+            "memory": {
+                "size": "64GB",
+                "type": "DDR4-ECC"
+            },
+            "features": {
+                "gpu": {
+                    "name": "Integrated GPU",
+                    "rops": 1,
+                    "tmus": 1,
+                    "cores": 128,
+                    "memory": {
+                        "size": "shared",
+                        "type": "shared"
+                    }
+                },
+                "network": "10G"
+            },
+            "disabled": False
+        }
+        response = self.client.get(
+            '/v2/limits/specs/',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data["count"], 1)
+        self.assertEqual(json.loads(json.dumps(response.data["results"][0])), results)
+        response = self.client.get(
+            '/v2/limits/specs/?keywords=unknown',
+        )
+        self.assertEqual(response.data["count"], 1)
+        response = self.client.get(
+            '/v2/limits/specs/?keywords=intel amd unknown',
+        )
+        self.assertEqual(response.data["count"], 1)
+        response = self.client.get(
+            '/v2/limits/specs/?keywords=noexists',
+        )
+        self.assertEqual(response.data["count"], 0)
+
+    def test_get_limits_plans(self, mock_requests):
+        response = self.client.get(
+            '/v2/limits/plans/',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data["count"], 6, response.data)
+        response = self.client.get(
+            '/v2/limits/plans/?cpu=1',
+        )
+        self.assertEqual(response.data["count"], 3, response.data)
+        response = self.client.get(
+            '/v2/limits/plans/?cpu=1&memory=1',
+        )
+        self.assertEqual(response.data["count"], 1, response.data)
+        response = self.client.get(
+            '/v2/limits/plans/?cpu=1&memory=1GB',
+        )
+        self.assertEqual(response.data["count"], 1, response.data)
+        response = self.client.get(
+            '/v2/limits/plans/?spec-id=std1',
+        )
+        self.assertEqual(response.data["count"], 6, response.data)
+        response = self.client.get(
+            '/v2/limits/plans/?spec-id=std2',
+        )
+        self.assertEqual(response.data["count"], 0, response.data)
+        # get one plan
+        response = self.client.get(
+            '/v2/limits/plans/std1.large.c1m1/',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.data["id"], "std1.large.c1m1")
diff --git a/rootfs/api/tests/test_limits.py b/rootfs/api/tests/test_limits.py
deleted file mode 100644
index 9a3bd538b..000000000
--- a/rootfs/api/tests/test_limits.py
+++ /dev/null
@@ -1,261 +0,0 @@
-import json
-import requests_mock
-
-from django.core.cache import cache
-from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
-
-from api.serializers import MEMLIMIT_MATCH
-from api.serializers import CPUSHARE_MATCH
-from api.tests import adapter, DryccTransactionTestCase
-
-User = get_user_model()
-
-
-@requests_mock.Mocker(real_http=True, adapter=adapter)
-class TestLimits(DryccTransactionTestCase):
-    """Tests setting and updating config values"""
-
-    fixtures = ['tests.json']
-
-    def setUp(self):
-        self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-
-    def tearDown(self):
-        # make sure every test has a clean slate for k8s mocking
-        cache.clear()
-
-    def test_memlimit_regex(self, mock_requests):
-        """Tests the regex for unit format used by "drycc limits:set --memory=<limit>"."""
-        self.assertTrue(MEMLIMIT_MATCH.match("2G"))
-        self.assertTrue(MEMLIMIT_MATCH.match("2M"))
-        self.assertTrue(MEMLIMIT_MATCH.match("20G"))
-        self.assertTrue(MEMLIMIT_MATCH.match("20M"))
-        self.assertTrue(MEMLIMIT_MATCH.match("2g"))
-        self.assertTrue(MEMLIMIT_MATCH.match("2m"))
-        self.assertTrue(MEMLIMIT_MATCH.match("20g"))
-        self.assertTrue(MEMLIMIT_MATCH.match("20m"))
-        self.assertFalse(MEMLIMIT_MATCH.match("0m"))
-        self.assertFalse(MEMLIMIT_MATCH.match("20MK"))
-        self.assertFalse(MEMLIMIT_MATCH.match("10"))
-        self.assertFalse(MEMLIMIT_MATCH.match("20gK"))
-        self.assertFalse(MEMLIMIT_MATCH.match("mb"))
-        self.assertFalse(MEMLIMIT_MATCH.match("0"))
-
-    def test_cpushare_regex(self, mock_requests):
-        """Tests the regex for unit format used by "drycc limits:set --cpu=<limit>"."""
-        self.assertTrue(CPUSHARE_MATCH.match("500m"))
-        self.assertTrue(CPUSHARE_MATCH.match("2"))
-        self.assertTrue(CPUSHARE_MATCH.match("12"))
-        self.assertFalse(CPUSHARE_MATCH.match("0.5"))
-        self.assertFalse(CPUSHARE_MATCH.match(".123"))
-        self.assertFalse(CPUSHARE_MATCH.match("1.123"))
-        self.assertFalse(CPUSHARE_MATCH.match("0"))
-        self.assertFalse(CPUSHARE_MATCH.match("20MK"))
-        self.assertFalse(CPUSHARE_MATCH.match("20gK"))
-        self.assertFalse(CPUSHARE_MATCH.match("m"))
-        self.assertFalse(CPUSHARE_MATCH.match("."))
-
-    def test_request_limit(self, mock_requests):
-        """
-        Test that limit is auto-created for a new app and that
-        limits can be updated using a PATCH
-        """
-        app_id = self.create_app()
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-
-        # check default limit
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertIn('memory', response.data)
-        self.assertEqual(response.data['memory'], {})
-        # regression test for https://github.com/drycc/drycc/issues/1563
-        self.assertNotIn('"', response.data['memory'])
-
-        # set an initial limit
-        mem = {'web': '1G'}
-        cpu = {'web': '1000m'}
-        body = {'memory': json.dumps(mem), 'cpu': json.dumps(cpu)}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        limit1 = response.data
-
-        # check memory limits
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertIn('memory', response.data)
-        memory = response.data['memory']
-        self.assertIn('web', memory)
-        self.assertEqual(memory['web'], '1G')
-
-        # check cpu limits
-        self.assertIn('cpu', response.data)
-        cpu = response.data['cpu']
-        self.assertIn('web', cpu)
-        self.assertEqual(cpu['web'], '1000m')
-
-        # set an additional value
-        body = {'memory': json.dumps({'worker': '512M'}), 'cpu': json.dumps({'worker': '2000m'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        limit2 = response.data
-        self.assertNotEqual(limit1['uuid'], limit2['uuid'])
-        memory = response.data['memory']
-        self.assertIn('worker', memory)
-        self.assertEqual(memory['worker'], '2G')
-        self.assertIn('web', memory)
-        self.assertEqual(memory['web'], '1G')
-        cpu = response.data['cpu']
-        self.assertIn('worker', cpu)
-        self.assertEqual(cpu['worker'], '2000m')
-        self.assertIn('web', cpu)
-        self.assertEqual(cpu['web'], '1000m')
-
-        # read the limit again
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        limit3 = response.data
-        self.assertEqual(limit2, limit3)
-        memory = response.data['memory']
-        self.assertIn('worker', memory)
-        self.assertEqual(memory['worker'], '2G')
-        self.assertIn('web', memory)
-        self.assertEqual(memory['web'], '1G')
-        cpu = response.data['cpu']
-        self.assertIn('worker', cpu)
-        self.assertEqual(cpu['worker'], '2000m')
-        self.assertIn('web', cpu)
-        self.assertEqual(cpu['web'], '1000m')
-
-        # regression test for https://github.com/drycc/drycc/issues/1613
-        # ensure that config:set doesn't wipe out previous limits
-        body = {'values': json.dumps({'NEW_URL2': 'http://localhost:8080/'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL2', response.data['values'])
-
-        # read the limit again
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        memory = response.data['memory']
-        self.assertIn('worker', memory)
-        self.assertEqual(memory['worker'], '2G')
-        self.assertIn('web', memory)
-        self.assertEqual(memory['web'], '1G')
-        cpu = response.data['cpu']
-        self.assertIn('worker', cpu)
-        self.assertEqual(cpu['worker'], '2000m')
-        self.assertIn('web', cpu)
-        self.assertEqual(cpu['web'], '1000m')
-
-        # add with requests/limits
-        body = {'memory': json.dumps({'db': '1G'}), 'cpu': json.dumps({'db': '1000m'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-
-        # read the limit again
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        memory = response.data['memory']
-        self.assertIn('worker', memory)
-        self.assertEqual(memory['worker'], '2G')
-        self.assertIn('web', memory)
-        self.assertEqual(memory['web'], '1G')
-        self.assertIn('db', memory)
-        self.assertEqual(memory['db'], '1G')
-        cpu = response.data['cpu']
-        self.assertIn('worker', cpu)
-        self.assertEqual(cpu['worker'], '2000m')
-        self.assertIn('web', cpu)
-        self.assertEqual(cpu['web'], '1000m')
-        self.assertIn('db', cpu)
-        self.assertEqual(cpu['db'], '1000m')
-
-        # replace one with requests/limits
-        body = {'memory': json.dumps({'web': '3G'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-
-        # read the limit again
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        memory = response.data['memory']
-        self.assertIn('worker', memory)
-        self.assertEqual(memory['worker'], '2G')
-        self.assertIn('web', memory)
-        self.assertEqual(memory['web'], '3G')
-        self.assertIn('db', memory)
-        self.assertEqual(memory['db'], '1G')
-
-        # replace one with requests/limits
-        body = {'cpu': json.dumps({'web': '3000m'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-
-        # read the limit again
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        cpu = response.data['cpu']
-        self.assertIn('worker', cpu)
-        self.assertEqual(cpu['worker'], '2000m')
-        self.assertIn('web', cpu)
-        self.assertEqual(cpu['web'], '3000m')
-        self.assertIn('db', cpu)
-        self.assertEqual(cpu['db'], '1000m')
-
-        # unset a value
-        body = {
-            'cpu': json.dumps({'worker': None}),
-            'memory': json.dumps({'worker': None})
-        }
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-        limit4 = response.data
-        self.assertNotEqual(limit3['uuid'], limit4['uuid'])
-        self.assertNotIn('worker', json.dumps(response.data['memory']))
-
-        # bad memory values
-        mem = {'web': '1Z'}
-        body = {'memory': json.dumps(mem)}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        # out of range
-        body = {'memory': json.dumps({'web': '64M'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        body = {'memory': json.dumps({'web': '1024G'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-
-        mem = {'w3&b': '1G'}
-        body = {'memory': json.dumps(mem)}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # bad cpu values
-        mem = {'web': '1G'}
-        body = {'cpu': json.dumps(mem)}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        # out of range
-        body = {'cpu': json.dumps({'web': '1000'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        body = {'cpu': json.dumps({'web': '125m'})}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
-
-        mem = {'w3&b': '1G'}
-        body = {'cpu': json.dumps(mem)}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-
-        # disallow put/patch/delete
-        response = self.client.put(url)
-        self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.patch(url)
-        self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
diff --git a/rootfs/api/tests/test_lock.py b/rootfs/api/tests/test_lock.py
new file mode 100644
index 000000000..146c8f037
--- /dev/null
+++ b/rootfs/api/tests/test_lock.py
@@ -0,0 +1,43 @@
+import time
+from api.tests import DryccTestCase
+from api.utils import CacheLock, DeployLock
+
+
+class TestLock(DryccTestCase):
+
+    def test_cache_lock(self):
+        key = f"test_key_1_{int(time.time())}"
+        lock1 = CacheLock(key)
+        lock2 = CacheLock(key)
+        timeout = 5
+        self.assertEqual(lock1.acquire(True, timeout), True)
+        self.assertEqual(lock2.acquire(False, timeout), False)
+        time.sleep(5)
+        start_time = time.time()
+        self.assertEqual(lock2.acquire(False, timeout), True)
+        self.assertEqual(lock1.acquire(True, timeout + 1), True)
+        self.assertTrue(time.time()-start_time > 5)
+        self.assertEqual(lock2.acquire(False, timeout), False)
+        lock1.release()
+        self.assertEqual(lock2.acquire(False, timeout), True)
+
+    def test_deploy_lock(self):
+        app_id = f"test_key_1_{int(time.time())}"
+        lock1 = DeployLock(app_id)
+        lock2 = DeployLock(app_id)
+        self.assertEqual(lock2.acquire(["web", "task"]), True)
+        self.assertEqual(lock1.acquire(["web"]), False)
+        self.assertEqual(lock1.acquire(["bing"]), True)
+        lock2.release(["web", "task"])
+        self.assertEqual(lock1.acquire(["web"]), True)
+        self.assertEqual(lock1.acquire(["task"]), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        lock2.release(["web", "task", "bing"])
+        self.assertEqual(lock1.acquire(["web", "task"]), True)
+        self.assertEqual(lock2.acquire(["web", "task"]), False)
+        self.assertEqual(lock2.acquire(["web", "bing"], True), True)
+        self.assertEqual(lock1.acquire(["web", "bing", "task"]), False)
+        self.assertEqual(lock1.acquire(["web", "bing", "task"], True), True)
+        lock2.release(["web", "bing", "task"])
+        self.assertEqual(lock1.acquire(["web"], True), True)
+        lock1.release(["web", "bing", "task"])
diff --git a/rootfs/api/tests/test_perm.py b/rootfs/api/tests/test_perm.py
index 9e4bc925e..96ec20e7a 100644
--- a/rootfs/api/tests/test_perm.py
+++ b/rootfs/api/tests/test_perm.py
@@ -1,213 +1,191 @@
 from django.test import tag
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
+
 from api.tests import DryccTestCase
+from api.models.workspace import WorkspaceMember
+from api.models.app import App
+
 
 User = get_user_model()
 
 
-class TestAppPerms(DryccTestCase):
+class TestWorkspacePerm(DryccTestCase):
 
-    fixtures = ['test_sharing.json']
+    fixtures = ['tests.json']
 
     @tag('auth')
     def setUp(self):
-        self.user = User.objects.get(username='autotest-1')
-        self.token = Token.objects.get(user=self.user).key
-        # Always have first user authed coming into tests
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
-        self.user2 = User.objects.get(username='autotest-2')
-        self.token2 = Token.objects.get(user=self.user2).key
-        self.user3 = User.objects.get(username='autotest-3')
-        self.token3 = Token.objects.get(user=self.user3).key
+        self.user2 = User.objects.get(username='autotest2')
+        self.token2 = self.get_or_create_token(self.user2)
 
     @tag('auth')
-    def test_create(self):
-        # check that user 1 sees her lone app and user 2's app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-        app_id = response.data['results'][0]['id']
-
-        # check that user 2 can only see his app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(len(response.data['results']), 1)
-        # check that user 2 can't see any of the app's builds, configs,
-        # containers, limits, or releases
-        for model in ['builds', 'config', 'pods', 'releases']:
-            response = self.client.get("/v2/apps/{}/{}/".format(app_id, model))
-            msg = "Failed: status '%s', and data '%s'" % (response.status_code, response.data)
-            self.assertEqual(response.status_code, 403, msg=msg)
-            self.assertEqual(response.data['detail'],
-                             'You do not have permission to perform this action.', msg=msg)
-
-        # TODO: test that git pushing to the app fails
-        # give user 2 permission to user 1's app
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.post(url, body)
+    def _create_workspace_and_app(self, ws_name='testws01', app_id='testapp01'):
+        response = self.client.post('/v2/workspaces', {
+            'name': ws_name,
+            'email': 'ws@example.com',
+        })
         self.assertEqual(response.status_code, 201, response.data)
 
-        # check that user 2 can see the app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-
-        # check that user 2 sees (empty) results now for builds, containers,
-        # and releases. (config and limit will still give 404s since we didn't
-        # push a build here.)
-        for model in ['builds', 'releases']:
-            response = self.client.get("/v2/apps/{}/{}/".format(app_id, model))
-            self.assertEqual(len(response.data['results']), 0)
-        # TODO:  check that user 2 can git push the app
+        response = self.client.post('/v2/apps', {
+            'id': app_id,
+            'workspace': ws_name,
+        })
+        self.assertEqual(response.status_code, 201, response.data)
+        return ws_name, app_id
 
     @tag('auth')
-    def test_create_errors(self):
-        # check that user 1 sees her lone app
-        response = self.client.get('/v2/apps')
-        app_id = response.data['results'][0]['id']
-
-        # check that user 2 can't create a permission
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
+    def test_workspace_member_can_access_app(self):
+        ws_name, app_id = self._create_workspace_and_app()
+
+        # non-member cannot access app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        response = self.client.get(f'/v2/apps/{app_id}')
+        self.assertEqual(response.status_code, 404, response.data)
 
-    @tag('auth')
-    def test_delete(self):
-        # give user 2 permission to user 1's app
-        response = self.client.get('/v2/apps')
-        app_id = response.data['results'][0]['id']
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
+        # add user2 into workspace, then user2 can access app
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # check that user 2 can see the app as well as his own
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
+        response = self.client.get(f'/v2/apps/{app_id}')
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
 
-        # delete permission to user 1's app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        url = "/v2/apps/{}/perms/{}".format(app_id, 'autotest-2')
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 204, response.data)
-        self.assertIsNone(response.data)
+    @tag('auth')
+    def test_non_admin_cannot_manage_workspace_members(self):
+        ws_name, _ = self._create_workspace_and_app(ws_name='testws02', app_id='testapp02')
 
-        # check that user 2 can only see his app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get('/v2/apps')
-        self.assertEqual(len(response.data['results']), 1)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # delete permission to user 1's app again, expecting an error
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 403)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.patch(
+            f'/v2/workspaces/{ws_name}/members/{self.user.username}',
+            {'role': 'viewer'},
+        )
+        self.assertEqual(response.status_code, 403, response.data)
 
     @tag('auth')
-    def test_list(self):
-        # check that user 1 sees her lone app and user 2's app
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
-        app_id = response.data['results'][0]['id']
+    def test_admin_can_manage_workspace_members(self):
+        ws_name, _ = self._create_workspace_and_app(ws_name='testws03', app_id='testapp03')
 
-        # create a new object permission
-        url = "/v2/apps/{}/perms".format(app_id)
-        body = {'username': 'autotest-2'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 201, response.data)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # list perms on the app
-        response = self.client.get("/v2/apps/{}/perms".format(app_id))
-        self.assertEqual(response.data, {'users': ['autotest-2']})
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        response = self.client.patch(
+            f'/v2/workspaces/{ws_name}/members/{self.user2.username}',
+            {'role': 'viewer'},
+        )
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['role'], 'viewer')
 
     @tag('auth')
-    def test_admin_can_list(self):
-        """Check that an administrator can list an app's perms"""
-        response = self.client.get('/v2/apps')
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 2)
+    def test_non_member_cannot_run_app(self):
+        _, app_id = self._create_workspace_and_app(ws_name='testws04', app_id='testapp04')
+
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.post(
+            f'/v2/apps/{app_id}/run',
+            {'command': 'echo hello'},
+        )
+        self.assertEqual(response.status_code, 404, response.data)
 
     @tag('auth')
-    def test_list_errors(self):
-        response = self.client.get('/v2/apps')
-        app_id = response.data['results'][0]['id']
+    def test_workspace_member_can_run_but_without_build_gets_business_error(self):
+        ws_name, app_id = self._create_workspace_and_app(ws_name='testws05', app_id='testapp05')
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # login as user 2, list perms on the app
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.get("/v2/apps/{}/perms".format(app_id))
-        self.assertEqual(response.status_code, 403)
+        response = self.client.post(
+            f'/v2/apps/{app_id}/run',
+            {'command': 'echo hello'},
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(
+            str(response.data['detail']),
+            'no build available, please deploy a release',
+        )
 
     @tag('auth')
-    def test_unauthorized_user_cannot_modify_perms(self):
-        """
-        An unauthorized user should not be able to modify other apps' permissions.
-
-        Since an unauthorized user should not know about the application at all, these
-        requests should return a 404.
-        """
-        app_id = 'autotest'
-        url = '/v2/apps'
-        body = {'id': app_id}
-        response = self.client.post(url, body)
-
-        url = '{}/{}/perms'.format(url, app_id)
-        body = {'username': self.user2.username}
+    def test_non_admin_cannot_update_workspace(self):
+        ws_name, _ = self._create_workspace_and_app(ws_name='testws06', app_id='testapp06')
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
+
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        response = self.client.patch(
+            f'/v2/workspaces/{ws_name}',
+            {'email': 'new@example.com'},
+        )
+        self.assertEqual(response.status_code, 403, response.data)
 
     @tag('auth')
-    def test_collaborator_cannot_share(self):
-        """
-        An collaborator should not be able to modify the app's permissions.
-        """
-        app_id = "autotest-1-app"
-        owner_token = self.token
-        collab = self.user2
-        collab_token = self.token2
-        url = '/v2/apps/{}/perms'.format(app_id)
-
-        # Share app with collaborator
-        body = {'username': collab.username}
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
-        response = self.client.post(url, body)
+    def test_non_admin_cannot_transfer_app(self):
+        ws_name, app_id = self._create_workspace_and_app(ws_name='testws07', app_id='testapp07')
+        response = self.client.post('/v2/workspaces', {
+            'name': 'testws08',
+            'email': 'ws2@example.com',
+        })
         self.assertEqual(response.status_code, 201, response.data)
 
-        # Collaborator should fail to share app
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
-        body = {'username': self.user3.username}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 403)
+        workspace = WorkspaceMember.objects.get(user=self.user, workspace__name=ws_name).workspace
+        WorkspaceMember.objects.get_or_create(
+            user=self.user2,
+            workspace=workspace,
+            defaults={'role': 'member'},
+        )
 
-        # Collaborator can list
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'testws08'},
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(
+            str(response.data['detail']),
+            'you must be an admin of the current workspace',
+        )
 
-        # Share app with user 3 for rest of tests
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + owner_token)
-        response = self.client.post(url, body)
+    @tag('auth')
+    def test_admin_can_transfer_app(self):
+        _, app_id = self._create_workspace_and_app(ws_name='testws09', app_id='testapp09')
+        response = self.client.post('/v2/workspaces', {
+            'name': 'testws10',
+            'email': 'ws10@example.com',
+        })
         self.assertEqual(response.status_code, 201, response.data)
 
-        self.client.credentials(HTTP_AUTHORIZATION='Token ' + collab_token)
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-
-        # Collaborator cannot delete other collaborator
-        url += "/{}".format(self.user3.username)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 403)
-
-        # Collaborator can delete themselves
-        url = '/v2/apps/{}/perms/{}'.format(app_id, collab.username)
-        response = self.client.delete(url)
+        response = self.client.patch(
+            f'/v2/apps/{app_id}',
+            {'workspace': 'testws10'},
+        )
         self.assertEqual(response.status_code, 204, response.data)
+
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.workspace.name, 'testws10')
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
index 3bc847517..371b8208c 100644
--- a/rootfs/api/tests/test_pods.py
+++ b/rootfs/api/tests/test_pods.py
@@ -4,12 +4,10 @@
 Run the tests with "./manage.py test api"
 """
 import os
-import json
 
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
 from unittest import mock
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
 from api.models.build import Build
@@ -30,7 +28,7 @@ class PodTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -41,13 +39,13 @@ def test_container_api_heroku(self, mock_requests):
         app_id = self.create_app()
 
         # should start with zero
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -61,7 +59,7 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         # test setting one proc type at a time
         body = {'web': 4}
         response = self.client.post(url, body)
@@ -71,45 +69,31 @@ def test_container_api_heroku(self, mock_requests):
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 6)
 
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         # ensure the structure field is up-to-date
         self.assertEqual(response.data['structure']['web'], 4)
         self.assertEqual(response.data['structure']['worker'], 2)
 
-        # test listing/retrieving container info
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 4)
-        self.assertEqual(len(response.data['results']), 4)
-
-        name = response.data['results'][0]['name']
-        url = "/v2/apps/{app_id}/pods/web/{name}".format(**locals())
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(response.data['count'], 1)
-        self.assertEqual(response.data['results'][0]['name'], name)
-
         # scale down
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         # test setting two proc types at a time
         body = {'web': 2, 'worker': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 3)
 
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         # ensure the structure field is up-to-date
@@ -117,17 +101,17 @@ def test_container_api_heroku(self, mock_requests):
         self.assertEqual(response.data['structure']['worker'], 1)
 
         # scale down to 0
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 0, 'worker': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
@@ -135,13 +119,13 @@ def test_container_api(self, mock_requests):
         app_id = self.create_app()
 
         # should start with zero
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'container',
@@ -151,53 +135,47 @@ def test_container_api(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 6}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 6)
 
-        url = "/v2/apps/{app_id}".format(**locals())
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 200, response.data)
-
-        # test listing/retrieving container info
-        url = "/v2/apps/{app_id}/pods/web".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 6)
 
         # scale down
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 3}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 3)
 
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
         # scale down to 0
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 0}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
 
@@ -205,13 +183,13 @@ def test_release(self, mock_requests):
         app_id = self.create_app()
 
         # should start with zero
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -225,19 +203,18 @@ def test_release(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        self.assertEqual(response.data['results'][0]['release'], 'v2')
+        self.assertPodContains(response.data['results'], app_id, "web", "v2")
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         # a web proctype must exist on the second build or else the container will be removed
         body = {
             'image': 'autotest/example',
@@ -250,58 +227,74 @@ def test_release(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(response.data['image'], body['image'])
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        self.assertEqual(response.data['results'][0]['release'], 'v3')
+        self.assertPodContains(response.data['results'], app_id, 'web', 'v3')
 
         # post new config
-        url = "/v2/apps/{app_id}/config".format(**locals())
-        body = {'values': json.dumps({'KEY': 'value'})}
+        url = f"/v2/apps/{app_id}/config"
+        body = {'values': [{'name': 'KEY', 'value': 'value', 'group': 'global'}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertEqual(len(response.data['results']), 1)
-        self.assertEqual(response.data['results'][0]['release'], 'v4')
+        self.assertPodContains(response.data['results'], app_id, "web", 'v4')
+
+        # describe pod
+        pod_name = response.data['results'][0]["name"]
+        response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # delete pods
+        response = self.client.delete(
+            f"/v2/apps/{app_id}/pods/",
+            {'pod_ids': pod_name}
+        )
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # describe no exists pod
+        pod_name = "no-exists-pod-name"
+        response = self.client.get(f"/v2/apps/{app_id}/pods/{pod_name}/describe/")
+        self.assertEqual(response.status_code, 400, response.data)
 
     def test_container_errors(self, mock_requests):
         app_id = self.create_app()
 
         # create a release so we can scale
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
-        build = Build.objects.create(owner=user, app=app, image="qwerty")
+        build = Build.objects.create(app=app, image="qwerty")
 
         # create an initial release
-        Release.objects.create(
+        release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
         )
-
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        # deploy
+        release.deploy()
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 'not_an_int'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
-        self.assertEqual(app.structure, {})
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.structure, {'web': 1})
 
         body = {'invalid': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
-        self.assertEqual(app.structure, {})
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.structure, {'web': 1})
 
     def test_container_str(self, mock_requests):
         """Test the text representation of a container."""
         app_id = self.create_app()
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -315,18 +308,20 @@ def test_container_str(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4, 'worker': 2}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
         # should start with zero
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 6)
         pods = response.data['results']
         for pod in pods:
+            if pod['state'] != 'up':
+                continue
             self.assertIn(pod['type'], ['web', 'worker'])
             self.assertEqual(pod['release'], 'v2')
             # pod name is auto generated so use regex
@@ -337,7 +332,7 @@ def test_pod_command_format(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -351,14 +346,14 @@ def test_pod_command_format(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
 
-        # verify that the app._get_command property got formatted
+        # verify that the release.get_deploy_args property got formatted
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 1)
 
@@ -369,20 +364,21 @@ def test_pod_command_format(self, mock_requests):
         self.assertRegex(pod['name'], app_id + '-web-[0-9]{8,10}-[a-z0-9]{5}')
 
         # verify commands
-        data = App.objects.get(id=app_id)
-        self.assertNotIn('{c_type}', data._get_command('web'))
+        release = App.objects.get(id=app_id).release_set.latest()
+
+        self.assertNotIn('{c_type}', release.get_deploy_args('web'))
 
     def test_scale_errors(self, mock_requests):
         app_id = self.create_app()
 
         # should start with zero
-        url = "/v2/apps/{app_id}/pods".format(**locals())
+        url = f"/v2/apps/{app_id}/pods"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['results']), 0)
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -396,7 +392,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # scale to a negative number
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': -1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -404,7 +400,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(app.structure["web"], 1)
 
         # scale to something other than a number
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 'one'}
         response = self.client.post(url, body)
         app = App.objects.get(id=app_id)
@@ -412,7 +408,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(app.structure["web"], 1)
 
         # scale to something other than a number
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': [1]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -420,7 +416,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual(app.structure["web"], 1)
 
         # scale with a non-existent proc type
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'foo': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -428,7 +424,7 @@ def test_scale_errors(self, mock_requests):
         self.assertEqual("foo" in app.structure, False)
 
         # scale up to an integer as a sanity check
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -437,7 +433,7 @@ def test_scale_errors(self, mock_requests):
 
         with mock.patch('scheduler.KubeHTTPClient.scale') as mock_kube:
             mock_kube.side_effect = KubeException('Boom!')
-            url = "/v2/apps/{app_id}/scale".format(**locals())
+            url = f"/v2/apps/{app_id}/ptypes/scale"
             response = self.client.post(url, {'web': 10})
             self.assertEqual(response.status_code, 204, response.data)
             app = App.objects.get(id=app_id)
@@ -448,13 +444,13 @@ def test_admin_can_manage_other_pods(self, mock_requests):
         manage it.
         """
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
 
         app_id = self.create_app()
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -468,7 +464,7 @@ def test_admin_can_manage_other_pods(self, mock_requests):
         self.assertEqual(response.status_code, 201, response.data)
 
         # login as admin, scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4, 'worker': 2}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -476,11 +472,9 @@ def test_admin_can_manage_other_pods(self, mock_requests):
     def test_scale_without_build_should_error(self, mock_requests):
         """A user should not be able to scale processes unless a build is present."""
         app_id = 'autotest'
-        url = '/v2/apps'
-        body = {'cluster': 'autotest', 'id': app_id}
-        response = self.client.post(url, body)
+        self.create_app(name=app_id)
 
-        url = '/v2/apps/{app_id}/scale'.format(**locals())
+        url = f'/v2/apps/{app_id}/ptypes/scale'
         body = {'web': '1'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -491,11 +485,9 @@ def test_command_good(self, mock_requests):
         """Test the default command for each container workflow"""
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
 
         # CNCF Buildpack app
         build = Build.objects.create(
-            owner=user,
             app=app,
             image="qwerty",
             procfile={
@@ -507,42 +499,43 @@ def test_command_good(self, mock_requests):
         )
 
         # create an initial release
-        Release.objects.create(
+        release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
         )
+        # deploy
+        release.deploy()
 
         # use `start web` for backwards compatibility with buildpacks
-        self.assertEqual(app._get_command('web'), [])
-        self.assertEqual(app._get_command('worker'), [])
+        self.assertEqual(release.get_deploy_args('web'), [])
+        self.assertEqual(release.get_deploy_args('worker'), [])
 
         # switch to container image app
         build.sha = ''
         build.save()
-        self.assertEqual(app._get_command('web'), ['node server.js'])
+        self.assertEqual(release.get_deploy_args('web'), ['node', 'server.js'])
 
         # switch to dockerfile app
         build.sha = 'european-swallow'
         build.dockerfile = 'dockerdockerdocker'
         build.save()
-        self.assertEqual(app._get_command('web'), ['node server.js'])
-        self.assertEqual(app._get_command('cmd'), [])
+        self.assertEqual(release.get_deploy_args('web'), ['node', 'server.js'])
+        self.assertEqual(release.get_deploy_args('cmd'), [])
 
         # ensure we can override the cmd process type in a Procfile
         build.procfile['cmd'] = 'node server.js'
         build.save()
-        self.assertEqual(app._get_entrypoint('cmd'), ['/bin/sh', '-c'])
-        self.assertEqual(app._get_command('cmd'), ['node server.js'])
-        self.assertEqual(app._get_entrypoint('worker'), ['/bin/sh', '-c'])
-        self.assertEqual(app._get_command('worker'), ['node worker.js'])
+        self.assertEqual(release.get_deploy_command('cmd'), [])
+        self.assertEqual(release.get_deploy_args('cmd'), ['node', 'server.js'])
+        self.assertEqual(release.get_deploy_command('worker'), [])
+        self.assertEqual(release.get_deploy_args('worker'), ['node', 'worker.js'])
 
         # for backwards compatibility if no Procfile is supplied
         build.procfile = {}
         build.save()
-        self.assertEqual(app._get_command('worker'), [])
+        self.assertEqual(release.get_deploy_args('worker'), [])
 
     def test_run_command_good(self, mock_requests):
         """Test the run command for each container workflow"""
@@ -551,7 +544,6 @@ def test_run_command_good(self, mock_requests):
 
         # dockerfile + procfile worflow
         build = Build.objects.create(
-            owner=self.user,
             app=app,
             image="qwerty",
             stack="heroku-18",
@@ -564,43 +556,42 @@ def test_run_command_good(self, mock_requests):
         )
 
         # create an initial release
-        Release.objects.create(
+        release = Release.objects.create(
             version=2,
-            owner=self.user,
             app=app,
             config=app.config_set.latest(),
             build=build
         )
+        # deploy
+        release.deploy()
 
         # create a run pod
-        url = "/v2/apps/{app_id}/run".format(**locals())
+        url = f"/v2/apps/{app_id}/run"
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.status_code, 204, response.data)
         app = App.objects.get(id=app_id)
-        self.assertEqual(app._get_entrypoint('web'), ['/bin/sh', '-c'])
+        self.assertEqual(release.get_deploy_command('web'), [])
 
         # docker image workflow
         build.dockerfile = ''
         build.sha = ''
         build.save()
-        url = "/v2/apps/{app_id}/run".format(**locals())
+        url = f"/v2/apps/{app_id}/run"
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.status_code, 204, response.data)
         app = App.objects.get(id=app_id)
-        self.assertEqual(app._get_entrypoint('cmd'), [])
-        self.assertEqual(app._get_entrypoint('run'), ['/bin/sh', '-c'])
+        self.assertEqual(release.get_deploy_command('cmd'), [])
+        self.assertEqual(release.get_deploy_command('run'), [])
 
         # procfile workflow
         build.sha = 'somereallylongsha'
         build.save()
-        url = "/v2/apps/{app_id}/run".format(**locals())
+        url = f"/v2/apps/{app_id}/run"
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
-        app = App.objects.get(id=app_id)
-        self.assertEqual(app._get_entrypoint('web'), ['web'])
+        self.assertEqual(response.status_code, 204, response.data)
 
     def test_run_not_fail_on_debug(self, mock_requests):
         """
@@ -613,7 +604,6 @@ def test_run_not_fail_on_debug(self, mock_requests):
 
         # dockerfile + procfile worflow
         build = Build.objects.create(
-            owner=self.user,
             app=app,
             image="qwerty",
             procfile={
@@ -625,31 +615,29 @@ def test_run_not_fail_on_debug(self, mock_requests):
         )
 
         # create an initial release
-        Release.objects.create(
+        release = Release.objects.create(
             version=2,
-            owner=self.user,
             app=app,
             config=app.config_set.latest(),
             build=build
         )
+        # deploy
+        release.deploy()
 
         # create a run pod
-        url = "/v2/apps/{app_id}/run".format(**locals())
+        url = f"/v2/apps/{app_id}/run"
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
-        app = App.objects.get(id=app_id)
-        self.assertEqual(app._get_entrypoint('web'), ['/bin/sh', '-c'])
+        self.assertEqual(response.status_code, 204, response.data)
+        self.assertEqual(release.get_deploy_command('web'), [])
 
     def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
         """Test that app info doesn't show transient "run" proctypes."""
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
-        user = User.objects.get(username='autotest')
 
         # dockerfile + procfile worflow
         build = Build.objects.create(
-            owner=user,
             app=app,
             image="qwerty",
             procfile={
@@ -663,26 +651,27 @@ def test_scaling_does_not_add_run_proctypes_to_structure(self, mock_requests):
         # create an initial release
         release = Release.objects.create(
             version=2,
-            owner=user,
             app=app,
             config=app.config_set.latest(),
             build=build
         )
+        # deploy
+        release.deploy()
 
         # create a run pod
-        url = "/v2/apps/{app_id}/run".format(**locals())
+        url = f"/v2/apps/{app_id}/run"
         body = {'command': 'echo hi'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.status_code, 204, response.data)
 
         # scale up
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 3}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
 
         # test that "run" proctype isn't in the app info returned
-        url = "/v2/apps/{app_id}".format(**locals())
+        url = f"/v2/apps/{app_id}"
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertNotIn('run', response.data['structure'])
@@ -696,7 +685,7 @@ def test_scale_with_unauthorized_user_returns_403(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        url = "/v2/apps/{app_id}/builds".format(**locals())
+        url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -704,11 +693,11 @@ def test_scale_with_unauthorized_user_returns_403(self, mock_requests):
         }
         response = self.client.post(url, body)
         unauthorized_user = User.objects.get(username='autotest2')
-        unauthorized_token = Token.objects.get(user=unauthorized_user).key
+        unauthorized_token = self.get_or_create_token(unauthorized_user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
 
         # scale up with unauthorized user
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 403)
@@ -721,7 +710,7 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         app_id = self.create_app()
 
         # post a new build
-        build_url = "/v2/apps/{app_id}/builds".format(**locals())
+        build_url = f"/v2/apps/{app_id}/build"
         body = {
             'image': 'autotest/example',
             'stack': 'heroku-18',
@@ -733,7 +722,7 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         }
         response = self.client.post(build_url, body)
 
-        url = "/v2/apps/{app_id}/scale".format(**locals())
+        url = f"/v2/apps/{app_id}/ptypes/scale"
         body = {'web': 4}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
@@ -748,51 +737,12 @@ def test_modified_procfile_from_build_removes_pods(self, mock_requests):
         }
         response = self.client.post(build_url, body)
         self.assertEqual(response.status_code, 201, response.data)
-
-        # make sure no pods are web
+        # check web pods
         application = App.objects.get(id=app_id)
         pods = application.list_pods(type='web')
         self.assertEqual(len(pods), 0)
-
-    def test_restart_pods(self, mock_requests):
-        app_id = self.create_app()
-
-        # post a new build
-        build_url = "/v2/apps/{app_id}/builds".format(**locals())
-        body = {
-            'image': 'autotest/example',
-            'stack': 'heroku-18',
-            'sha': 'a'*40,
-            'procfile': {
-                'web': 'node server.js',
-                'worker': 'node worker.js'
-            }
-        }
-        response = self.client.post(build_url, body)
-
-        url = "/v2/apps/{app_id}/scale".format(**locals())
-        body = {'web': 4, 'worker': 8}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # setup app object
-        application = App.objects.get(id=app_id)
-
-        # restart all pods
-        response = self.client.post('/v2/apps/{}/pods/restart'.format(app_id))
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # restart only the workers
-        response = self.client.post('/v2/apps/{}/pods/worker/restart'.format(app_id))
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # restart only the web
-        response = self.client.post('/v2/apps/{}/pods/web/restart'.format(app_id))
-        self.assertEqual(response.status_code, 204, response.data)
-
-        # restart only one of the web pods
-        pods = application.list_pods(type='web')
-        self.assertEqual(len(pods), 4)
+        pods = application.list_pods(type='worker')
+        self.assertEqual(len(pods), 0)
 
     def test_list_pods_failure(self, mock_requests):
         """
@@ -805,6 +755,6 @@ def test_list_pods_failure(self, mock_requests):
             with mock.patch('scheduler.resources.pod.Pod.get') as kube_pods:
                 kube_pod.side_effect = KubeException('boom!')
                 kube_pods.side_effect = KubeException('boom!')
-                url = "/v2/apps/{app_id}/pods".format(**locals())
+                url = f"/v2/apps/{app_id}/pods"
                 response = self.client.get(url)
                 self.assertEqual(response.status_code, 503, response.data)
diff --git a/rootfs/api/tests/test_procfile.py b/rootfs/api/tests/test_procfile.py
new file mode 100644
index 000000000..cf4f3eb8b
--- /dev/null
+++ b/rootfs/api/tests/test_procfile.py
@@ -0,0 +1,94 @@
+from rest_framework.test import APITestCase
+from rest_framework import serializers
+from api.serializers import validate_ptype, BuildSerializer, ServiceSerializer
+
+
+class ProcfileTypeTest(APITestCase):
+
+    def assertException(self, method, exception=Exception, *args, **kwargs):
+        try:
+            method(*args, **kwargs)
+        except BaseException as e:
+            if not isinstance(e, exception):
+                raise AssertionError(
+                    "Did not throw the expected exception, "
+                    f"actual: {type(e)}, expect: {exception}"
+                )
+        else:
+            raise AssertionError(
+                f"Did not throw the expected exception, args: {args}, kwargs: {kwargs}")
+
+    def test_ptype_error(self):
+        for ptype in ["w", "we", "-a", "we-new-", "w" * 64]:
+            self.assertException(
+                validate_ptype, serializers.ValidationError, ptype)
+
+    def test_ptype_ok(self):
+        self.assertEqual(validate_ptype("web"), "web")
+        self.assertEqual(validate_ptype("w" * 63), "w" * 63)
+        self.assertEqual(validate_ptype("web-new-be"), "web-new-be")
+
+    def test_dryccfile_ptype(self):
+        dryccfile_1 = {
+            "pipeline": {
+                "web.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "web",
+                    "build": {
+                        "docker": "Dockerfile",
+                    },
+                    "deploy": {
+                        "args": ["python", "-m", "http.server", "5000"],
+                    },
+                },
+                "worker.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "worker",
+                    "build": {
+                        "docker": "Dockerfile.task",
+                    },
+                    "deploy": {
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    },
+                }
+            }
+        }
+        validate_dryccfile = BuildSerializer().validate_dryccfile
+        self.assertEqual(validate_dryccfile(dryccfile_1), dryccfile_1)
+        dryccfile_1["pipeline"]["w.yaml"] = {
+            "kind": "pipeline",
+            "ptype": "w",
+            "build": {"docker": "Dockerfile.task"},
+        }
+        self.assertException(
+                validate_dryccfile, serializers.ValidationError, dryccfile_1)
+        del dryccfile_1["pipeline"]["w.yaml"]
+
+        dryccfile_1["pipeline"]["w.yaml"] = {
+            "kind": "pipeline",
+            "ptype": "w",
+            "build": {"docker": "Dockerfile.task"},
+            "deploy": {"args": ["python", "-m", "http.server", "5000"]},
+        }
+        self.assertException(
+                validate_dryccfile, serializers.ValidationError, dryccfile_1)
+        dryccfile_1["pipeline"]["w.yaml"]
+
+        dryccfile_1["pipeline"]["wnew.yaml"] = {
+            "kind": "pipeline",
+            "ptype": "w-new-",
+            "build": {"docker": "Dockerfile.task"},
+            "deploy": {"args": ["python", "-m", "http.server", "5000"]},
+        }
+        self.assertException(
+                validate_dryccfile, serializers.ValidationError, dryccfile_1)
+
+    def test_staticmethod_ptype(self):
+        s_validate_ptype = ServiceSerializer().validate_ptype
+        for ptype in ["w", "we", "-a", "we-new-", "w" * 64]:
+            self.assertException(
+                s_validate_ptype, serializers.ValidationError, ptype)
+        self.assertEqual(s_validate_ptype("web"), "web")
+        self.assertEqual(s_validate_ptype("w" * 63), "w" * 63)
+        self.assertEqual(s_validate_ptype("web-new-be"), "web-new-be")
diff --git a/rootfs/api/tests/test_ptypes.py b/rootfs/api/tests/test_ptypes.py
new file mode 100644
index 000000000..15a509bee
--- /dev/null
+++ b/rootfs/api/tests/test_ptypes.py
@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for the Drycc api app.
+
+Run the tests with "./manage.py test api"
+"""
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+
+from api.models.app import App
+
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class PtypesTest(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        self.app = App.objects.get(id=self.create_app())
+
+    def tearDown(self):
+        # make sure every test has a clean slate for k8s mocking
+        cache.clear()
+
+    def build_deploy(self, app_id):
+        # post a new build with procfile
+        url = "/v2/apps/{app_id}/build".format(app_id=app_id)
+        body = {
+            'image': 'autotest/example',
+            'sha': 'a'*40,
+            'stack': 'heroku-18',
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+
+    def test_ptype(self, mock_requests):
+        """
+        Test that config is auto-created for a new app and that
+        config can be updated using a PATCH
+        """
+        app_id = self.create_app()
+        self.build_deploy(app_id)
+
+        # list ptypes deployment
+        url = f"/v2/apps/{app_id}/ptypes"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+
+        # describe ptype deployment
+        url = f"/v2/apps/{app_id}/ptypes/{app_id}-web/describe"
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+
+        ptype = "no-exists-ptype-name"
+        response = self.client.get(f"/v2/apps/{app_id}/ptypes/{ptype}/describe/")
+        self.assertEqual(response.status_code, 400, response.data)
+
+    def test_restart_ptypes(self, mock_requests):
+        app_id = self.create_app()
+
+        # post a new build
+        build_url = f"/v2/apps/{app_id}/build"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(build_url, body)
+
+        url = f"/v2/apps/{app_id}/ptypes/scale"
+        body = {'web': 4, 'worker': 8}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # setup app object
+        application = App.objects.get(id=app_id)
+
+        # restart all ptypes deployments
+        response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id))
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # restart web and workers ptype deployments
+        body = {"ptypes": "web,worker"}
+        response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id), body)
+        self.assertEqual(response.status_code, 204, response.data)
+
+        # restart invalid ptypes
+        body = {"ptypes": "web1"}
+        response = self.client.post('/v2/apps/{}/ptypes/restart'.format(app_id), body)
+        self.assertEqual(response.status_code, 400, response.data)
+
+        # restart only one of the web ptype deployments
+        pods = application.list_pods(type='web')
+        self.assertEqual(len(pods), 4)
diff --git a/rootfs/api/tests/test_registry.py b/rootfs/api/tests/test_registry.py
index 757d90ef9..0be570856 100644
--- a/rootfs/api/tests/test_registry.py
+++ b/rootfs/api/tests/test_registry.py
@@ -3,7 +3,6 @@
 
 from django.core.cache import cache
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 
 from api.tests import adapter, DryccTransactionTestCase
 
@@ -18,7 +17,7 @@ class TestRegistry(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -32,26 +31,26 @@ def test_registry(self, mock_requests):
         app_id = self.create_app()
 
         # check default
-        url = '/v2/apps/{app_id}/config'.format(**locals())
+        url = f'/v2/apps/{app_id}/config'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('registry', response.data)
         self.assertEqual(response.data['registry'], {})
 
         # set some registry information without PORT
-        body = {'registry': json.dumps({'username': 'bob'})}
+        body = {'registry': {'web': {'username': 'bob'}}}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(response.status_code, 201, response.data)
         registry1 = response.data
 
         # set required PORT
-        body = {'values': json.dumps({'PORT': '80'})}
+        body = {'values': [{"name": "PORT", "value": "80", "group": "global"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry1 = response.data
 
-        # set some registry information
-        body = {'registry': json.dumps({'username': 'bob'})}
+        # no change error
+        body = {'registry': {'web': {'username': 'bobb'}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry1 = response.data
@@ -60,45 +59,45 @@ def test_registry(self, mock_requests):
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('registry', response.data)
-        registry = response.data['registry']
+        registry = response.data['registry']['web']
         self.assertIn('username', registry)
-        self.assertEqual(registry['username'], 'bob')
+        self.assertEqual(registry['username'], 'bobb')
 
         # set an additional value
-        # set them upper case, internally it should translate to lower
-        body = {'registry': json.dumps({'PASSWORD': 's3cur3pw1'})}
+        body = {'registry': {'web': {'password': 's3cur3pw1'}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry2 = response.data
         self.assertNotEqual(registry1['uuid'], registry2['uuid'])
-        registry = response.data['registry']
+        registry = response.data['registry']['web']
         self.assertIn('password', registry)
         self.assertEqual(registry['password'], 's3cur3pw1')
         self.assertIn('username', registry)
-        self.assertEqual(registry['username'], 'bob')
+        self.assertEqual(registry['username'], 'bobb')
 
         # read the registry information again
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         registry3 = response.data
         self.assertEqual(registry2, registry3)
-        registry = response.data['registry']
+        registry = response.data['registry']['web']
         self.assertIn('password', registry)
         self.assertEqual(registry['password'], 's3cur3pw1')
         self.assertIn('username', registry)
-        self.assertEqual(registry['username'], 'bob')
+        self.assertEqual(registry['username'], 'bobb')
 
         # unset a value
-        body = {'registry': json.dumps({'password': None})}
+        body = {'registry': {'web': {'password': None}}}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         registry4 = response.data
         self.assertNotEqual(registry3['uuid'], registry4['uuid'])
         self.assertNotIn('password', json.dumps(response.data['registry']))
 
-        # bad registry key values
-        body = {'registry': json.dumps({'pa$$w0rd': 'woop'})}
+        # key error
+        body = {'registry': {'web': {'pa$$w0rd': 'woop'}}}
         response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 400, response.data)
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
@@ -107,8 +106,6 @@ def test_registry(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
 
     def test_registry_deploy(self, mock_requests):
         """
@@ -118,21 +115,25 @@ def test_registry_deploy(self, mock_requests):
 
         # Set mandatory PORT
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
-            {'values': json.dumps({'PORT': '4999'})}
+            f'/v2/apps/{app_id}/config',
+            {'values': [{"name": "PORT", "value": "4999", "group": "global"}]}
         )
 
         # Set registry information
-        body = {'registry': json.dumps({
-            'username': 'bob',
-            'password': 's3cur3pw1'
-        })}
+        body = {
+            'registry': {
+                'web': {
+                    'username': 'bob',
+                    'password': 's3cur3pw1'
+                }
+            }
+        }
         response = self.client.post(
-            '/v2/apps/{app_id}/config'.format(**locals()),
+            f'/v2/apps/{app_id}/config',
             body
         )
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('username', response.data['registry'])
-        self.assertIn('password', response.data['registry'])
-        self.assertEqual(response.data['registry']['username'], 'bob')
-        self.assertEqual(response.data['registry']['password'], 's3cur3pw1')
+        self.assertIn('username', response.data['registry']['web'])
+        self.assertIn('password', response.data['registry']['web'])
+        self.assertEqual(response.data['registry']['web']['username'], 'bob')
+        self.assertEqual(response.data['registry']['web']['password'], 's3cur3pw1')
diff --git a/rootfs/api/tests/test_release.py b/rootfs/api/tests/test_release.py
index 8becf1af0..88ef6d6e1 100644
--- a/rootfs/api/tests/test_release.py
+++ b/rootfs/api/tests/test_release.py
@@ -9,12 +9,12 @@
 from django.core.cache import cache
 from django.test.utils import override_settings
 from unittest import mock
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
+from api.models.build import Build
 from api.models.release import Release
 from scheduler import KubeHTTPException
-from api.exceptions import DryccException
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
 
@@ -30,7 +30,7 @@ class ReleaseTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -44,18 +44,20 @@ def test_release(self, mock_requests):
         """
         app_id = self.create_app()
         # check that updating config rolls a new release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
         # check to see that an initial release was created
-        url = '/v2/apps/{app_id}/releases'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         # account for the config release as well
         self.assertEqual(response.data['count'], 2)
-        url = '/v2/apps/{app_id}/releases/v1'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases/v1'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         release1 = response.data
@@ -63,7 +65,7 @@ def test_release(self, mock_requests):
         self.assertIn('build', response.data)
         self.assertEqual(release1['version'], 1)
         # check to see that a new release was created
-        url = '/v2/apps/{app_id}/releases/v2'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases/v2'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         release2 = response.data
@@ -72,14 +74,13 @@ def test_release(self, mock_requests):
         self.assertEqual(release1['build'], release2['build'])
         self.assertEqual(release2['version'], 2)
         # check that updating the build rolls a new release
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
-        build_config = json.dumps({'PATH': 'bin:/usr/local/bin:/usr/bin:/bin'})
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(response.data['image'], body['image'])
         # check to see that a new release was created
-        url = '/v2/apps/{app_id}/releases/v3'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases/v3'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         release3 = response.data
@@ -87,7 +88,7 @@ def test_release(self, mock_requests):
         self.assertNotEqual(release2['build'], release3['build'])
         self.assertEqual(release3['version'], 3)
         # check that we can fetch a previous release
-        url = '/v2/apps/{app_id}/releases/v2'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases/v2'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         release2 = response.data
@@ -95,7 +96,7 @@ def test_release(self, mock_requests):
         self.assertNotEqual(release2['build'], release3['build'])
         self.assertEqual(release2['version'], 2)
         # disallow post/put/patch/delete
-        url = '/v2/apps/{app_id}/releases'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases'
         response = self.client.post(url)
         self.assertEqual(response.status_code, 405, response.content)
         response = self.client.put(url)
@@ -106,58 +107,240 @@ def test_release(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.content)
         return release3
 
+    def test_conditions(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/build"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker-test1': 'node worker.js'
+            }
+        }
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        url = f'/v2/apps/{app_id}/releases/'
+        response = self.client.get(url)
+        self.assertEqual(len(response.data['results'][0]['conditions']), 1)
+
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.side_effect = Exception('Boom!')
+            url = f"/v2/apps/{app_id}/build"
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            url = f'/v2/apps/{app_id}/releases/'
+            response = self.client.get(url)
+            self.assertEqual(
+                response.data['results'][0]['conditions'][0]['exception'], 'Exception: Boom!')
+
+    def test_get_image(self, mock_requests):
+        app_id = self.create_app()
+        url = f"/v2/apps/{app_id}/build"
+        body = {
+            'image': 'registry.drycc.cc:5555/autotest/example:git-fadf1231',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'dryccfile': {
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["python myworker.py"],
+                        },
+                    },
+                    "worker-1.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-1",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "image": "worker",
+                        },
+                    },
+                    "worker-2.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-2",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {},
+                    },
+                    "worker-3.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-3",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": "web"
+                        },
+                    },
+                    "worker-4.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker-4",
+                        "run": {"command": ["./deployment-tasks.sh"], "image": "worker"},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": "registry.drycc.cc:7070/myapp/web:git-123fsa1",
+                        },
+                    },
+                },
+            },
+        }
+        default_image = 'registry.drycc.cc:5555/autotest/example:git-fadf1231'
+        worker_image = "registry.drycc.cc:5555/autotest/example:git-fadf1231-worker"
+        worker_4_image = "registry.drycc.cc:7070/myapp/web:git-123fsa1"
+
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+            app = App.objects.get(id=app_id)
+            release_obj = app.release_set.filter(version=2)[0]
+            runners = release_obj.get_runners(["web"])
+            self.assertEqual(len(runners), 1, release_obj.build.dryccfile)
+            self.assertEqual(
+                runners[0]["image"],
+                worker_image, release_obj.build.dryccfile)
+            self.assertEqual(
+                release_obj.get_deploy_image("web"),
+                default_image, release_obj.build.dryccfile)
+            self.assertEqual(
+                release_obj.get_deploy_image("worker"),
+                worker_image, release_obj.build.dryccfile)
+            self.assertEqual(
+                release_obj.get_deploy_image("worker-1"),
+                worker_image, release_obj.build.dryccfile)
+            self.assertEqual(
+                release_obj.get_deploy_image("worker-2"),
+                default_image, release_obj.build.dryccfile)
+            self.assertEqual(
+                release_obj.get_deploy_image("worker-3"),
+                default_image, release_obj.build.dryccfile)
+            self.assertEqual(
+                release_obj.get_deploy_image("worker-4"),
+                worker_4_image, release_obj.build.dryccfile)
+
     def test_response_data(self, mock_requests):
         app_id = self.create_app()
-        body = {'values': json.dumps({'NEW_URL': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL", "value": "http://localhost:8080/", "group": "global"}]
+        }
         url = '/v2/apps/{}/config'.format(app_id)
         config_response = self.client.post(url, body)
         url = '/v2/apps/{}/releases/v2'.format(app_id)
         response = self.client.get(url)
         for key in response.data.keys():
-            self.assertIn(key, ['uuid', 'owner', 'created', 'updated', 'app', 'build', 'config',
-                                'summary', 'canary', 'version', 'failed', 'exception'])
+            self.assertIn(key, ['uuid', 'created', 'updated', 'app', 'build', 'config',
+                                'summary', 'version', 'state', 'failed', 'conditions',
+                                'exception', 'deployed_ptypes'])
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'build': None,
             'config': uuid.UUID(config_response.data['uuid']),
-            'summary': '{} added NEW_URL'.format(self.user.username),
+            'summary': '{} added values NEW_URL'.format(self.user.username),
             'version': 2
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, response.data | expected)
+
+    def test_release_deploy(self, mock_requests):
+        app_id = self.create_app()
+        # try to rollback with only 1 release extant, expecting 400
+        url = f"/v2/apps/{app_id}/releases/deploy/"
+        # app.deploy
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.return_value = None
+            response = self.client.post(url, {"ptypes": "web"})
+            self.assertEqual(response.status_code, 400)
+
+        # post a new build
+        build_url = f"/v2/apps/{app_id}/build"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'procfile': {
+                'web': 'node server.js',
+                'worker': 'node worker.js'
+            }
+        }
+        response = self.client.post(build_url, body)
+        self.assertEqual(response.status_code, 201)
+        # app.deploy
+        with mock.patch('api.models.app.App.deploy') as mock_deploy:
+            mock_deploy.return_value = None
+            response = self.client.post(url, {"ptypes": "web"})
+            self.assertEqual(response.status_code, 204)
+
+        url = f'/v2/apps/{app_id}/releases/?ptypes=web'
+        response = self.client.get(url)
+        self.assertEqual(response.data['count'], 1)
+        url = f'/v2/apps/{app_id}/releases/?ptypes=web,worker'
+        response = self.client.get(url)
+        self.assertEqual(response.data['count'], 1)
+        url = f'/v2/apps/{app_id}/releases/?ptypes=web,worker,noexists'
+        response = self.client.get(url)
+        self.assertEqual(response.data['count'], 0)
 
     def test_release_rollback(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
         # try to rollback with only 1 release extant, expecting 400
-        url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
+        url = f"/v2/apps/{app_id}/releases/rollback/"
         response = self.client.post(url)
         self.assertEqual(response.status_code, 400, response.data)
         self.assertEqual(response.data, {'detail': 'version cannot be below 0'})
         self.assertEqual(response.get('content-type'), 'application/json')
         # update the build to roll a new release
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # update config to roll another release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f'/v2/apps/{app_id}/config'
+        values = [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        body = {'values': values}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # create another release with a different build
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
-        body = {'image': 'autotest/example:canary', 'stack': 'container'}
+        url = f'/v2/apps/{app_id}/build'
+        body = {'image': 'autotest/example:latest', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         # rollback and check to see that a 5th release was created
         # with the build and config of release #3
-        url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
+        url = f"/v2/apps/{app_id}/releases/rollback/"
         response = self.client.post(url)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(Release.objects.count(), 5)
-        release1 = Release.objects.get(app=app, version=1)
-        release2 = Release.objects.get(app=app, version=2)
+        Release.objects.get(app=app, version=1)
+        Release.objects.get(app=app, version=2)
         release3 = Release.objects.get(app=app, version=3)
         release4 = Release.objects.get(app=app, version=4)
         release5 = Release.objects.get(app=app, version=5)
@@ -167,23 +350,23 @@ def test_release_rollback(self, mock_requests):
         self.assertEqual(release5.build, release3.build)
         self.assertEqual(release5.config.values, release3.config.values)
         # double-check to see that the current build and config is the same as v3
-        self.assertEqual(release5.build.image, 'autotest/example')
-        self.assertEqual(release5.config.values, {'NEW_URL1': 'http://localhost:8080/'})
+        self.assertEqual(release5.get_deploy_image(PTYPE_WEB), 'autotest/example')
+        self.assertEqual(release5.config.values, values)
         # try to rollback to v1 and verify that the rollback failed
         # (v1 is an initial release with no build)
-        url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
+        url = f"/v2/apps/{app_id}/releases/rollback/"
         body = {'version': 1}
         response = self.client.post(url, body)
         self.assertContains(response, 'Cannot roll back to initial release.', status_code=400)
         # roll back to v2 so we can verify config gets rolled back too
-        url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
+        url = f"/v2/apps/{app_id}/releases/rollback/"
         body = {'version': 2}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(Release.objects.count(), 6)
         release6 = Release.objects.get(app=app, version=6)
-        self.assertEqual(release6.build.image, 'autotest/example')
-        self.assertEqual(release6.config.values, {})
+        self.assertEqual(release6.get_deploy_image(PTYPE_WEB), 'autotest/example')
+        self.assertEqual(release6.config.values, [])
 
     def test_release_str(self, mock_requests):
         """Test the text representation of a release."""
@@ -199,31 +382,33 @@ def test_release_summary(self, mock_requests):
         # check that the release has push and env change messages
         self.assertIn('autotest deployed ', release.summary)
         # add config, confirm that config objects are in the summary
-        url = '/v2/apps/{app.id}/config'.format(**locals())
+        url = f'/v2/apps/{app.id}/config'
         body = {
-            'values': json.dumps({'FOO': 'bar'}),
+            'values': [{"name": "FOO", "value": "bar", "group": "global"}],
         }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertEqual(
-            'autotest added FOO',
+            'autotest added values FOO',
             app.release_set.latest().summary)
 
     def test_admin_can_create_release(self, mock_requests):
         """If a non-user creates an app, an admin should be able to create releases."""
         user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user).key
+        token = self.get_or_create_token(user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
         app_id = self.create_app()
         # check that updating config rolls a new release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
         # check to see that an initial release was created
-        url = '/v2/apps/{app_id}/releases'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         # account for the config release as well
@@ -239,20 +424,22 @@ def test_unauthorized_user_cannot_modify_release(self, mock_requests):
         app_id = self.create_app()
 
         # push a new build
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'test', 'stack': 'container'}
         response = self.client.post(url, body)
 
         # update config to roll a new release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         unauthorized_user = User.objects.get(username='autotest2')
-        unauthorized_token = Token.objects.get(user=unauthorized_user).key
+        unauthorized_token = self.get_or_create_token(unauthorized_user)
 
         # try to rollback
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + unauthorized_token)
-        url = '/v2/apps/{app_id}/releases/rollback/'.format(**locals())
+        url = f'/v2/apps/{app_id}/releases/rollback/'
         response = self.client.post(url)
         self.assertEqual(response.status_code, 403)
 
@@ -261,10 +448,9 @@ def test_release_rollback_failure(self, mock_requests):
         Cause an Exception in app.deploy to cause a release.delete
         """
         app_id = self.create_app()
-        app = App.objects.get(id=app_id)
 
         # deploy app to get a build
-        url = "/v2/apps/{}/builds".format(app_id)
+        url = "/v2/apps/{}/build".format(app_id)
         body = {'image': 'autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
@@ -272,7 +458,9 @@ def test_release_rollback_failure(self, mock_requests):
 
         # update config to roll a new release
         url = '/v2/apps/{}/config'.format(app_id)
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
@@ -282,18 +470,21 @@ def test_release_rollback_failure(self, mock_requests):
             url = "/v2/apps/{}/releases/rollback/".format(app_id)
             body = {'version': 2}
             response = self.client.post(url, body)
-            self.assertEqual(response.status_code, 400, response.data)
-            self.assertEqual(app.release_set.latest().version, 4)
+            self.assertEqual(response.status_code, 201, response.data)
+            data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
+            self.assertEqual(data["results"][0]["state"], "crashed", data)
 
         # update config to roll a new release
         url = '/v2/apps/{}/config'.format(app_id)
-        body = {'values': json.dumps({'NEW_URL2': 'http://localhost:8080/'})}
+        body = {
+            'values': [{"name": "NEW_URL2", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
         # try to rollback to v4 and verify that the rollback failed
         # (v4 is a failed release)
-        url = "/v2/apps/{app_id}/releases/rollback/".format(**locals())
+        url = f"/v2/apps/{app_id}/releases/rollback/"
         body = {'version': 4}
         response = self.client.post(url, body)
         self.assertContains(response, 'Cannot roll back to failed release.', status_code=400)
@@ -317,7 +508,9 @@ def json(self):
                 url = "/v2/apps/{}/releases/rollback/".format(app_id)
                 body = {'version': 2}
                 response = self.client.post(url, body)
-                self.assertEqual(response.status_code, 400, response.data)
+                self.assertEqual(response.status_code, 201, response.data)
+                data = self.client.get(f"/v2/apps/{app_id}/releases/", body).json()
+                self.assertEqual(data["results"][0]["state"], "crashed", data)
 
     def test_release_unset_config(self, mock_requests):
         """
@@ -327,8 +520,8 @@ def test_release_unset_config(self, mock_requests):
         app_id = self.create_app()
 
         # check that updating config rolls a new release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'cpu': json.dumps({'cmd': None})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {'limits': json.dumps({'cmd': None})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 422, response.data)
 
@@ -341,15 +534,19 @@ def test_release_no_change(self, mock_requests):
         app_id = self.create_app()
 
         # check that updating config rolls a new release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        self.assertIn('NEW_URL1', response.data['values'])
+        self.assertEqual(body['values'], response.data['values'])
 
         # trigger identical release
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {
+            'values': [{"name": "NEW_URL1", "value": "http://localhost:8080/", "group": "global"}]
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 409, response.data)
 
@@ -360,34 +557,56 @@ def test_release_get_port(self, mock_requests):
         app_id = self.create_app()
         app = App.objects.get(id=app_id)
 
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         body = {'sha': '123456', 'image': 'autotest/example', 'stack': 'heroku-18'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
 
         # when app is not routable, then it still return 5000
-        self.assertEqual(release.get_port(), 5000)
-
-        # when a buildpack type, default to 5000
-        self.assertEqual(release.get_port(), 5000)
+        self.assertEqual(release.get_port('web'), 5000)
 
         # switch to a dockerfile app or else it'll automatically default to 5000
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
-        body = {'image': 'autotest/example', 'stack': 'container'}
+        url = f'/v2/apps/{app_id}/build'
+        body = {
+            'image': 'autotest/example',
+            'stack': 'container',
+            'procfile': {
+                'web': 'node server.js',
+                'task': 'node worker.js'
+            },
+        }
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
 
-        url = '/v2/apps/{app_id}/config'.format(**locals())
-        body = {'values': json.dumps({'PORT': '8080'})}
+        url = f'/v2/apps/{app_id}/config'
+        body = {'values': [{"name": "PORT", "value": "8080", "group": "global"}]}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
+        self.assertEqual(release.get_port('web'), 8080)
 
-        # check that the port number returned is an int, not a string
-        self.assertEqual(release.get_port(), 8080)
+        url = f'/v2/apps/{app_id}/config'
+        body = {'values': [{"name": "PORT", "value": "9000", "ptype": "web"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('web'), 9000)
 
-        # TODO(bacongobbler): test dockerfile ports
+        # not web procfile
+        self.assertEqual(release.get_port('task'), 8080)
+        url = f'/v2/apps/{app_id}/config'
+        body = {'values': [{"name": "PORT", "value": "9000", "group": "global"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('task'), 9000)
+        # set task port
+        body = {'values': [{"name": "PORT", "value": "9001", "ptype": "task"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('task'), 9001)
 
     @override_settings(DRYCC_DEPLOY_HOOK_URLS=['http://drycc.rocks'])
     @mock.patch('api.models.release.logger')
@@ -395,33 +614,33 @@ def test_deploy_hooks_logged(self, mock_requests, mock_logger):
         """
         Verifies that a configured deploy hook is dumped into the logs when a release is created.
         """
-        app_id = 'foo'
+        app_id = 'foooo'
         body = {'sha': '123456', 'image': 'autotest/example', 'stack': 'heroku-18'}
 
-        mr_rocks = mock_requests.post('http://drycc.rocks?app={app_id}&user={self.user.username}&sha=&release=v1&release_summary={self.user.username}+created+initial+release'.format(**locals()))  # noqa
+        mr_rocks = mock_requests.post(f'http://drycc.rocks?app={app_id}&user={self.user.username}&sha=&release=v1&release_summary={self.user.username}+created+initial+release')  # noqa
         self.create_app(app_id)
         # check app logs
-        exp_msg = "[{app_id}]: Sent deploy hook to http://drycc.rocks".format(**locals())
-        mock_logger.log.has_calls(logging.INFO, exp_msg)
+        exp_msg = f"[{app_id}]: Sent deploy hook to http://drycc.rocks"
+        mock_logger.log.assert_any_call(logging.INFO, exp_msg)
         self.assertTrue(mr_rocks.called)
         self.assertEqual(mr_rocks.call_count, 1)
 
         # override DRYCC_DEPLOY_HOOK_URLS again, ensuring that the new deploy hooks get the same
         # treatment
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         with self.settings(DRYCC_DEPLOY_HOOK_URLS=['http://drycc.ninja', 'http://cat.dog']):
-            mr_ninja = mock_requests.post("http://drycc.ninja?app={app_id}&user={self.user.username}&sha=123456&release=v2&release_summary={self.user.username}+deployed+123456".format(**locals()))  # noqa
-            mr_catdog = mock_requests.post("http://cat.dog?app={app_id}&user={self.user.username}&sha=123456&release=v2&release_summary={self.user.username}+deployed+123456".format(**locals()))  # noqa
+            mr_ninja = mock_requests.post(f"http://drycc.ninja?app={app_id}&user={self.user.username}&sha=123456&release=v2&release_summary={self.user.username}+deployed+123456")  # noqa
+            mr_catdog = mock_requests.post(f"http://cat.dog?app={app_id}&user={self.user.username}&sha=123456&release=v2&release_summary={self.user.username}+deployed+123456")  # noqa
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
 
             # check app logs
-            exp_msg = "[{app_id}]: Sent deploy hook to http://drycc.ninja".format(**locals())
-            mock_logger.log.has_calls(logging.INFO, exp_msg)
+            exp_msg = f"[{app_id}]: Sent deploy hook to http://drycc.ninja"
+            mock_logger.log.assert_any_call(logging.INFO, exp_msg)
             self.assertTrue(mr_ninja.called)
             self.assertEqual(mr_ninja.call_count, 1)
-            exp_msg = "[{app_id}]: Sent deploy hook to http://cat.dog".format(**locals())
-            mock_logger.log.has_calls(logging.INFO, exp_msg)
+            exp_msg = f"[{app_id}]: Sent deploy hook to http://cat.dog"
+            mock_logger.log.assert_any_call(logging.INFO, exp_msg)
             self.assertTrue(mr_catdog.called)
             self.assertEqual(mr_catdog.call_count, 1)
         sha = '2345678'
@@ -430,18 +649,18 @@ def test_deploy_hooks_logged(self, mock_requests, mock_logger):
         with self.settings(DRYCC_DEPLOY_HOOK_URLS=['http://cat.ninja', 'http://drycc.dog']):
             def raise_callback(request, context):
                 raise requests.ConnectionError('poop')
-            mr_ninja = mock_requests.post("http://cat.ninja?app={app_id}&user={self.user.username}&sha={sha}&release=v3&release_summary={self.user.username}+deployed+{sha}".     format(**locals()), text=raise_callback)  # noqa
-            mr_catdog = mock_requests.post("http://drycc.dog?app={app_id}&user={self.user.username}&sha={sha}&release=v3&release_summary={self.user.username}+deployed+{sha}".       format(**locals()))  # noqa
+            mr_ninja = mock_requests.post(f"http://cat.ninja?app={app_id}&user={self.user.username}&sha={sha}&release=v3&release_summary={self.user.username}+deployed+{sha}", text=raise_callback)  # noqa
+            mr_catdog = mock_requests.post(f"http://drycc.dog?app={app_id}&user={self.user.username}&sha={sha}&release=v3&release_summary={self.user.username}+deployed+{sha}")  # noqa
             response = self.client.post(url, body)
             self.assertEqual(response.status_code, 201, response.data)
 
             # check app logs
-            exp_msg = "[{app_id}]: An error occurred while sending the deploy hook to http://cat.ninja: poop".format(**locals())  # noqa
-            mock_logger.log.has_calls(logging.ERROR, exp_msg)
+            exp_msg = f"[{app_id}]: An error occurred while sending the deploy hook to http://cat.ninja: poop"  # noqa
+            mock_logger.log.assert_any_call(logging.ERROR, exp_msg)
             self.assertTrue(mr_ninja.called)
             self.assertEqual(mr_ninja.call_count, 1)
-            exp_msg = "[{app_id}]: Sent deploy hook to http://drycc.dog".format(**locals())
-            mock_logger.log.has_calls(logging.INFO, exp_msg)
+            exp_msg = f"[{app_id}]: Sent deploy hook to http://drycc.dog"
+            mock_logger.log.assert_any_call(logging.INFO, exp_msg)
             self.assertTrue(mr_catdog.called)
             self.assertEqual(mr_catdog.call_count, 1)
 
@@ -476,8 +695,8 @@ def raise_callback(request, context):
             self.assertEqual(response.status_code, 201, response.data)
 
             # check app logs
-            exp_msg = "[{app_id}]: Sent deploy hook to {hook_url}".format(**locals())
-            mock_logger.log.has_calls(logging.INFO, exp_msg)
+            exp_msg = f"[{app_id}]: Sent deploy hook to {hook_url}"
+            mock_logger.log.assert_any_call(logging.INFO, exp_msg)
             self.assertTrue(mr_terminator.called)
             self.assertEqual(mr_terminator.call_count, 1)
 
@@ -489,50 +708,199 @@ def test_release_external_registry(self, mock_requests):
         app_id = self.create_app()
 
         # set the required port for external registries
-        body = {'values': json.dumps({'PORT': '3000'})}
+        body = {
+            'values': [{"name": "PORT", "value": "3000", "group": "global"}]
+        }
         config_response = self.client.post('/v2/apps/{}/config'.format(app_id), body)
         self.assertEqual(config_response.status_code, 201, config_response.data)
 
         app = App.objects.get(id=app_id)
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
+        url = f'/v2/apps/{app_id}/build'
         body = {'image': 'test/autotest/example', 'stack': 'container'}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
+        self.assertEqual(release.get_port('web'), 3000)
+        self.assertEqual(release.get_deploy_image(PTYPE_WEB), 'test/autotest/example')
 
-        self.assertEqual(release.get_port(), 3000)
-
-        self.assertEqual(release.image, 'test/autotest/example')
+        url = f'/v2/apps/{app_id}/config'
+        body = {'values': [{"name": "PORT", "value": "9000", "ptype": "web"}]}
+        response = self.client.post(url, body)
+        self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.get_port('web'), 9000)
 
-    @override_settings(REGISTRY_LOCATION="off-cluster")
-    def test_release_external_registry_no_port(self, mock_requests):
-        """
-        Test that an exception is raised when registry if off-cluster but
-        no port is provided.
-        """
+    @override_settings(DRYCC_PILELINE_RUN_TIMEOUT=600)
+    def test_run(self, mock_requests):
         app_id = self.create_app()
-
         app = App.objects.get(id=app_id)
-        url = '/v2/apps/{app_id}/builds'.format(**locals())
-        body = {'image': 'test/autotest/example', 'stack': 'container'}
-        response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-
-        with self.assertRaises(
-            DryccException,
-            msg='PORT needs to be set in the config when using a private registry'
-        ):
-            release = app.release_set.latest()
-            release.get_port()
-
-        # Set routable to false and port should be None instead of error
-        response = self.client.post(
-            '/v2/apps/{app.id}/settings'.format(**locals()),
-            {'routable': False}
-        )
-        self.assertEqual(response.status_code, 201, response.data)
-        self.assertFalse(app.appsettings_set.latest().routable)
+        dryccfile = {
+            "pipeline": {
+                "web.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "web",
+                    "run": {
+                        "args": ["sleep", "60s"],
+                        "image": "registry.drycc.cc/drycc/base:bookworm-web",
+                    },
+                    "deploy": {
+                        "image": "registry.drycc.cc/drycc/python-dev",
+                        "args": ["python", "-m", "http.server", "5000"]
+                    },
+                },
+                "task.yaml": {
+                    "kind": "pipeline",
+                    "ptype": "task",
+                    "run": {
+                        "args": ["sleep", "60s"],
+                        "image": "registry.drycc.cc/drycc/base:bookworm-task",
+                        "timeout": 3000
+                    },
+                    "deploy": {
+                        "image": "docker.io/library/nginx:mainline-bookworm-perl",
+                        "command": ["sleep"],
+                        "args": ["infinity"]
+                    },
+                },
+            },
+        }
+        build = Build.objects.create(
+            app=app, image="qwerty", procfile={},
+            sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
+        release = Release.objects.create(
+            version=2, app=app, config=app.config_set.latest(),
+            build=build, state="succeed")
+        runners = release.get_runners(["web"])
+        self.assertEqual(len(runners), 1)
+        self.assertEqual(runners[0]["timeout"], 600)
+
+        runners = release.get_runners(["task"])
+        self.assertEqual(len(runners), 1)
+        self.assertEqual(runners[0]["timeout"], 3000)
+
+        # change timeout
+        dryccfile['pipeline']['web.yaml']['run']['timeout'] = 3600
+        dryccfile['pipeline']['web.yaml']['deploy'] = {
+            'image': 'registry.drycc.cc/drycc/python-dev:latest',
+        }
+        build = Build.objects.create(
+            app=app, image="qwerty", procfile={},
+            sha='african-swallow', dockerfile={}, dryccfile=dryccfile)
+        release = Release.objects.create(
+            version=3, app=app, config=app.config_set.latest(),
+            build=build, state="succeed")
+        runners = release.get_runners(["web"])
+        self.assertEqual(len(runners), 1)
+        self.assertEqual(runners[0]["timeout"], 3600)
+
+    def test_release_diff_ptypes(self, mock_requests):
+        app_id = self.create_app()
+        app = App.objects.get(id=app_id)
+        url = f"/v2/apps/{app_id}/build"
+        run_image = "127.0.0.1:7070/myapp/run:git-123fsa1"
+        web_image = "127.0.0.1:7070/myapp/web:git-123fsa1"
+        worker_image = "127.0.0.1:7070/myapp/worker:git-123fsa1"
+        body = {
+            'image': 'autotest/example',
+            'stack': 'heroku-18',
+            'sha': 'a'*40,
+            'dryccfile': {
+                "pipeline": {
+                    "web.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "web",
+                        "build": {
+                            "docker": "Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["bundle exec puma -C config/puma.rb"],
+                            "image": web_image
+                        },
+                    },
+                    "worker.yaml": {
+                        "kind": "pipeline",
+                        "ptype": "worker",
+                        "build": {
+                            "docker": "worker/Dockerfile",
+                            "arg": {
+                                "FOO": "bar",
+                                "RAILS_ENV": "development",
+                            },
+                        },
+                        "run": {"command": ["./deployment-tasks.sh"], "image": run_image},
+                        "deploy": {
+                            "command": ["bash", "-c"],
+                            "args": ["python myworker.py"],
+                            "image": worker_image
+                        },
+                    },
+                },
+            },
+        }
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["web", "worker"])
 
+        body['dryccfile']['config'] = {"g1": {"G1": "g1"}}
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, [])
+        # change worker env
+        body['dryccfile']['pipeline']['worker.yaml']['config'] = ["g1"]
+        body['dryccfile']['config'] = {"g1": {"G1": "g1"}}
+
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["worker"])
+        # env cover, only change web
+        body['dryccfile']['config'] = {"global": {"G1": "g1"}}
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["web"])
+        # add a global env
+        body['dryccfile']['config'] = {"global": {"G2": "g2"}}
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
+        release = app.release_set.latest()
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(set(ptypes), set(["web", "worker"]))
+
+        # change web env
+        body['dryccfile']['pipeline']['web.yaml']['env'] = {"ENV1": "env1"}
+        with mock.patch('scheduler.resources.pod.Pod.watch') as mock_kube:
+            mock_kube.return_value = ['up', 'down']
+            response = self.client.post(url, body)
+            self.assertEqual(response.status_code, 201, response.data)
         release = app.release_set.latest()
-        port = release.get_port()
-        self.assertIsNone(port)
+        self.assertEqual(release.failed, False)
+        ptypes = release.diff_ptypes(release.ptypes)
+        self.assertEqual(ptypes, ["web"])
diff --git a/rootfs/api/tests/test_resource.py b/rootfs/api/tests/test_resource.py
index e011dabb1..37af38ef9 100644
--- a/rootfs/api/tests/test_resource.py
+++ b/rootfs/api/tests/test_resource.py
@@ -6,8 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
-from rest_framework.authtoken.models import Token
 from api.tests import adapter, DryccTransactionTestCase
 from api.exceptions import DryccException
 import requests_mock
@@ -23,13 +21,11 @@ class ResourceTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         self.app_id = self.create_app()
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
@@ -37,6 +33,26 @@ def test_resources_create(self, mock_requests):
         """Test that the serialized response contains only relevant data."""
         app_id = self.create_app()
 
+        # name regex error
+        name1 = 'mysql_'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': name1, 'plan': 'mysql:5.6'}
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name2 = '-mysql'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': name2, 'plan': 'mysql:5.6'}
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name3 = 'mysql.8'
+        response = self.client.post(
+            '/v2/apps/{}/resources'.format(app_id),
+            data={'name': name3, 'plan': 'mysql:5.6'}
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+
         response = self.client.post(
             '/v2/apps/{}/resources'.format(app_id),
             data={'name': 'mysql', 'plan': 'mysql:5.6'}
@@ -45,16 +61,15 @@ def test_resources_create(self, mock_requests):
 
         for key in response.data:
             self.assertIn(key,
-                          ['uuid', 'owner', 'created', 'updated', 'app', 'plan',
+                          ['uuid', 'created', 'updated', 'app', 'plan',
                            'options', 'data', 'status', 'binding', 'name'])
 
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'name': 'mysql',
             'plan': 'mysql:5.6'
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, expected | response.data)
 
     def test_resources_list(self, mock_requests):
         """
@@ -139,7 +154,7 @@ def call_command(self, *args, **kwargs):
         from django.core.management import call_command
         out = StringIO()
         call_command(
-            "measure_resources",
+            "upload_resource_usage",
             *args,
             stdout=out,
             stderr=StringIO(),
@@ -147,7 +162,7 @@ def call_command(self, *args, **kwargs):
         )
         return out.getvalue()
 
-    def test_measure_resources(self, *args, **kwargs):
+    def test_upload_resource_usage(self, *args, **kwargs):
         # create
         app_id = self.create_app()
         data = [
@@ -157,3 +172,15 @@ def test_measure_resources(self, *args, **kwargs):
             self.client.post('/v2/apps/{}/resources'.format(app_id), data=_)
         out = self.call_command()
         self.assertIn(out, "done\n")
+
+    def test_services(self, *args, **kwargs):
+        # Get
+        url = '/v2/resources/services/'
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
+
+    def test_plans(self, *args, **kwargs):
+        # Get
+        url = '/v2/resources/services/mysql/plans/'
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200, response.data)
diff --git a/rootfs/api/tests/test_service.py b/rootfs/api/tests/test_service.py
index 0518399af..85f8f4b3a 100644
--- a/rootfs/api/tests/test_service.py
+++ b/rootfs/api/tests/test_service.py
@@ -1,9 +1,6 @@
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
-from rest_framework.authtoken.models import Token
 
-from api.models.app import App
 from api.tests import DryccTransactionTestCase
 from api.tests.test_gateway import RouteTest
 
@@ -18,7 +15,7 @@ class ServiceTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         self.test_route = RouteTest()
         self.test_route.user = self.user
@@ -43,7 +40,7 @@ def test_service_basic_ops(self):
                 'port': 5000,
                 'protocol': 'UDP',
                 'target_port': 5000,
-                'procfile_type': 'test'
+                'ptype': 'test'
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -52,17 +49,16 @@ def test_service_basic_ops(self):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['services']), 1)
         expected0 = {
-            "domain": "%s-%s.%s.svc.%s" % (
-                app_id, "test", app_id, settings.KUBERNETES_CLUSTER_DOMAIN),
+            "domain": "%s-%s.%s.svc" % (app_id, "test", app_id),
             "ports": [{
                 'name': "%s-%s-%s-%s" % (app_id, "test", 'udp', 5000),
                 'port': 5000,
                 'protocol': 'UDP',
                 'targetPort': 5000,
             }],
-            "procfile_type": "test"
+            "ptype": "test"
         }
-        self.assertDictContainsSubset(expected0, response.data['services'][0])
+        self.assertEqual(response.data['services'][0], expected0 | response.data['services'][0])
         # port is occupied
         response = self.client.post(
             '/v2/apps/{}/services'.format(app_id),
@@ -70,15 +66,14 @@ def test_service_basic_ops(self):
                 'port': 5000,
                 'protocol': 'UDP',
                 'target_port': 5000,
-                'procfile_type': 'test'
+                'ptype': 'test'
             }
         )
         self.assertEqual(response.status_code, 400, response.data)
 
         # add new port
         expected1 = {
-            "domain": "%s-%s.%s.svc.%s" % (
-                app_id, "test", app_id, settings.KUBERNETES_CLUSTER_DOMAIN),
+            "domain": "%s-%s.%s.svc" % (app_id, "test", app_id),
             "ports": [
                 {
                     'name': "%s-%s-%s-%s" % (app_id, "test", 'udp', 5000),
@@ -93,7 +88,7 @@ def test_service_basic_ops(self):
                     'targetPort': 6000,
                 }
             ],
-            "procfile_type": "test"
+            "ptype": "test"
         }
         response = self.client.post(
             '/v2/apps/{}/services'.format(app_id),
@@ -101,13 +96,14 @@ def test_service_basic_ops(self):
                 'port': 6000,
                 'protocol': 'TCP',
                 'target_port': 6000,
-                'procfile_type': 'test'
+                'ptype': 'test'
             }
         )
         self.assertEqual(response.status_code, 204, response.data)
         response = self.client.get('/v2/apps/{}/services'.format(app_id))
         self.assertEqual(response.status_code, 200, response.data)
-        self.assertDictContainsSubset(expected1, response.data['services'][0])
+        self.assertEqual(
+            response.data['services'][0], expected1 | response.data['services'][0])
 
         # create 2nd service
         response = self.client.post(
@@ -116,7 +112,7 @@ def test_service_basic_ops(self):
                 'port': 5000,
                 'protocol': 'UDP',
                 'target_port': 5000,
-                'procfile_type': 'test2'
+                'ptype': 'test2'
             }
         )
         self.assertEqual(response.status_code, 201, response.data)
@@ -125,64 +121,41 @@ def test_service_basic_ops(self):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertEqual(len(response.data['services']), 2)
         expected2 = {
-            "domain": "%s-%s.%s.svc.%s" % (
-                app_id, "test2", app_id, settings.KUBERNETES_CLUSTER_DOMAIN),
+            "domain": "%s-%s.%s.svc" % (app_id, "test2", app_id),
             "ports": [{
                 'name': "%s-%s-%s-%s" % (app_id, "test2", 'udp', 5000),
                 'port': 5000,
                 'protocol': 'UDP',
                 'targetPort': 5000,
             }],
-            "procfile_type": "test2"
+            "ptype": "test2"
         }
-        self.assertDictContainsSubset(expected2, response.data['services'][0])
-        self.assertDictContainsSubset(expected1, response.data['services'][1])
+        self.assertEqual(response.data['services'][0], expected2 | response.data['services'][0])
+        self.assertEqual(response.data['services'][1], expected1 | response.data['services'][1])
         # delete port
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test', "protocol": "TCP", "port": 6000}
+            {'ptype': 'test', "protocol": "TCP", "port": 6000}
         )
         response = self.client.get('/v2/apps/{}/services'.format(app_id))
-        self.assertDictContainsSubset(expected0, response.data['services'][1])
+        self.assertEqual(response.data['services'][1], expected0 | response.data['services'][1])
         # delete 1st
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test', "protocol": "UDP", "port": 5000}
+            {'ptype': 'test', "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 204, response.data)
         # delete 2nd
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test2',  "protocol": "UDP", "port": 5000}
+            {'ptype': 'test2',  "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 204, response.data)
         response = self.client.get('/v2/apps/{}/services'.format(app_id))
-        self.assertEqual(response.data["services"], [])
+        self.assertEqual(len(response.data["services"]), 0)
         # delete non-existing (1st again)
         response = self.client.delete(
             '/v2/apps/{}/services'.format(app_id),
-            {'procfile_type': 'test', "protocol": "UDP", "port": 5000}
+            {'ptype': 'test', "protocol": "UDP", "port": 5000}
         )
         self.assertEqual(response.status_code, 404, response.data)
-
-    def test_app_settings_change_canaries(self):
-        procfile_type, app_id, _, _, _, _ = self.test_route.test_route_attach()
-        # Add canaries
-        response = self.client.post(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': [procfile_type]}
-        )
-        self.assertEqual(response.status_code, 201, response.data)
-        app = App.objects.get(id=app_id)
-        response = app._scheduler.svc.get(app_id)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.json()["items"]), 2)
-        # Remove canaries to false
-        response = self.client.delete(
-            f'/v2/apps/{app_id}/settings',
-            {'canaries': [procfile_type]}
-        )
-        self.assertEqual(response.status_code, 204, response.data)
-        response = app._scheduler.svc.get(app_id)
-        self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.json()["items"]), 1)
diff --git a/rootfs/api/tests/test_tags.py b/rootfs/api/tests/test_tags.py
index e770ff814..dd6e81438 100644
--- a/rootfs/api/tests/test_tags.py
+++ b/rootfs/api/tests/test_tags.py
@@ -3,7 +3,6 @@
 
 from django.core.cache import cache
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 
 from api.tests import adapter, DryccTransactionTestCase
 
@@ -18,7 +17,7 @@ class TestTags(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
@@ -32,14 +31,14 @@ def test_tags(self, mock_requests):
         app_id = self.create_app()
 
         # check default
-        url = '/v2/apps/{app_id}/config'.format(**locals())
+        url = f'/v2/apps/{app_id}/config'
         response = self.client.get(url)
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('tags', response.data)
         self.assertEqual(response.data['tags'], {})
 
         # set some tags
-        body = {'tags': json.dumps({'environ': 'dev'})}
+        body = {'tags': json.dumps({'web': {'environ': 'dev'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         tags1 = response.data
@@ -49,20 +48,19 @@ def test_tags(self, mock_requests):
         self.assertEqual(response.status_code, 200, response.data)
         self.assertIn('tags', response.data)
         tags = response.data['tags']
-        self.assertIn('environ', tags)
-        self.assertEqual(tags['environ'], 'dev')
+        self.assertIn('web', tags)
+        self.assertEqual(tags['web']['environ'], 'dev')
 
         # set an additional value
-        body = {'tags': json.dumps({'rack': '1'})}
+        body = {'tags': json.dumps({'web': {'rack': '1'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         tags2 = response.data
         self.assertNotEqual(tags1['uuid'], tags2['uuid'])
         tags = response.data['tags']
-        self.assertIn('rack', tags)
-        self.assertEqual(tags['rack'], '1')
-        self.assertIn('environ', tags)
-        self.assertEqual(tags['environ'], 'dev')
+        self.assertIn('web', tags)
+        self.assertEqual(tags['web']['rack'], '1')
+        self.assertEqual(tags['web']['environ'], 'dev')
 
         # read the limit again
         response = self.client.get(url)
@@ -70,13 +68,12 @@ def test_tags(self, mock_requests):
         tags3 = response.data
         self.assertEqual(tags2, tags3)
         tags = response.data['tags']
-        self.assertIn('rack', tags)
-        self.assertEqual(tags['rack'], '1')
-        self.assertIn('environ', tags)
-        self.assertEqual(tags['environ'], 'dev')
+        self.assertIn('web', tags)
+        self.assertEqual(tags['web']['rack'], '1')
+        self.assertEqual(tags['web']['environ'], 'dev')
 
         # unset a value
-        body = {'tags': json.dumps({'rack': None})}
+        body = {'tags': json.dumps({'web': {'rack': None}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
         tags4 = response.data
@@ -84,16 +81,16 @@ def test_tags(self, mock_requests):
         self.assertNotIn('rack', json.dumps(response.data['tags']))
 
         # set valid values
-        body = {'tags': json.dumps({'kubernetes.io/hostname': '172.17.8.100'})}
+        body = {'tags': json.dumps({'web': {'kubernetes.io/hostname': '172.17.8.100'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        body = {'tags': json.dumps({'is.valid': 'is-also_valid'})}
+        body = {'tags': json.dumps({'web': {'is.valid': 'is-also_valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        body = {'tags': json.dumps({'host.the-name.com/is.valid': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.the-name.com/is.valid': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 201, response.data)
-        body = {'tags': json.dumps({'host.the-name.com/does.no.exist': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.the-name.com/does.no.exist': 'valid'}})}
         response = self.client.post(url, body)
         self.assertContains(
             response,
@@ -102,23 +99,23 @@ def test_tags(self, mock_requests):
         )
 
         # set invalid values
-        body = {'tags': json.dumps({'valid': 'in\nvalid'})}
+        body = {'tags': json.dumps({'web': {'valid': 'in\nvalid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'host.name.com/notvalid-': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.name.com/notvalid-': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'valid': 'invalid.'})}
+        body = {'tags': json.dumps({'web': {'valid': 'invalid.'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'host.name.com/,not.valid': 'valid'})}
+        body = {'tags': json.dumps({'web': {'host.name.com/,not.valid': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
         long_tag = 'a' * 300
-        body = {'tags': json.dumps({'{}/not.valid'.format(long_tag): 'valid'})}
+        body = {'tags': json.dumps({'web': {'{}/not.valid'.format(long_tag): 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
-        body = {'tags': json.dumps({'this&foo.com/not.valid': 'valid'})}
+        body = {'tags': json.dumps({'web': {'this&foo.com/not.valid': 'valid'}})}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 400, response.data)
 
@@ -127,5 +124,3 @@ def test_tags(self, mock_requests):
         self.assertEqual(response.status_code, 405, response.data)
         response = self.client.patch(url)
         self.assertEqual(response.status_code, 405, response.data)
-        response = self.client.delete(url)
-        self.assertEqual(response.status_code, 405, response.data)
diff --git a/rootfs/api/tests/test_tls.py b/rootfs/api/tests/test_tls.py
index 5cc35dd65..3ca2baf0e 100644
--- a/rootfs/api/tests/test_tls.py
+++ b/rootfs/api/tests/test_tls.py
@@ -2,9 +2,10 @@
 
 from django.core.cache import cache
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 
 from api.models.app import App
+from api.models.base import PTYPE_WEB
+from api.models.tls import TLS
 from api.tests import adapter, DryccTransactionTestCase
 
 User = get_user_model()
@@ -18,13 +19,19 @@ class TestTLS(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
 
     def tearDown(self):
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
+    def change_certs_auto(self, app_id, enabled):
+        data = {'certs_auto_enabled': enabled}
+        response = self.client.post(f'/v2/apps/{app_id}/tls', data)
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.data.get('certs_auto_enabled'), enabled, response.data)
+
     def test_tls_enforced(self, mock_requests):
         """
         Test that tls redirection can be enforced
@@ -34,7 +41,7 @@ def test_tls_enforced(self, mock_requests):
 
         data = {'https_enforced': True}
         response = self.client.post(
-            '/v2/apps/{app_id}/tls'.format(**locals()),
+            f'/v2/apps/{app_id}/tls',
             data)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertTrue(response.data.get('https_enforced'), response.data)
@@ -42,14 +49,14 @@ def test_tls_enforced(self, mock_requests):
 
         data = {'https_enforced': False}
         response = self.client.post(
-            '/v2/apps/{app_id}/tls'.format(**locals()),
+            f'/v2/apps/{app_id}/tls',
             data)
         self.assertEqual(response.status_code, 201, response.data)
         self.assertFalse(app.tls_set.latest().https_enforced)
 
         # when the same data is sent again, a 409 is returned
         conflict_response = self.client.post(
-            '/v2/apps/{app_id}/tls'.format(**locals()),
+            f'/v2/apps/{app_id}/tls',
             data)
         self.assertEqual(conflict_response.status_code, 409, conflict_response.data)
         self.assertFalse(app.tls_set.latest().https_enforced)
@@ -60,10 +67,24 @@ def test_tls_enforced(self, mock_requests):
         # sending bad data returns a 400
         data['https_enforced'] = "test"
         response = self.client.post(
-            '/v2/apps/{app_id}/tls'.format(**locals()),
+            f'/v2/apps/{app_id}/tls',
             data)
         self.assertEqual(response.status_code, 400, response.data)
 
+    def test_tls_events(self, mock_requests):
+        app_id = self.create_app()
+
+        response = self.client.post(
+            '/v2/apps/{}/domains'.format(app_id),
+            {'domain': 'test-domain.example.com', 'ptype': PTYPE_WEB}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.change_certs_auto(app_id, True)
+        tls = TLS.objects.get(app__id=app_id)
+        tls.refresh_certificate_to_k8s()
+        response = self.client.get('/v2/apps/{}/tls'.format(app_id))
+        self.assertEqual(len(response.json()["events"]), 3)
+
     def test_tls_created_on_app_create(self, mock_requests):
         """
         Ensure that a TLS object is created for an App with default values.
diff --git a/rootfs/api/tests/test_token.py b/rootfs/api/tests/test_token.py
new file mode 100644
index 000000000..d73dbf945
--- /dev/null
+++ b/rootfs/api/tests/test_token.py
@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+Unit tests for the Drycc api app.
+
+Run the tests with "./manage.py test api"
+"""
+from django.contrib.auth import get_user_model
+from api.models.base import Token
+from api.tests import adapter, DryccTransactionTestCase
+import requests_mock
+
+User = get_user_model()
+
+
+@requests_mock.Mocker(real_http=True, adapter=adapter)
+class TokenTest(DryccTransactionTestCase):
+    """Tests setting and updating config values"""
+
+    fixtures = ['tests.json']
+
+    def test_token(self, *args, **kwargs):
+        user = User.objects.get(username='autotest')
+        key1 = self.get_or_create_token(user)
+        url = '/v2/tokens'
+        response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(key1))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()['count'], 1)
+        token = Token.objects.create(
+            owner=user,
+            alias="workflow-cli",
+            oauth={
+                "access_token": "test",
+                "expires_in": 3600 * 24 * 7,
+                "token_type": "Bearer",
+                "scope": "openid",
+                "refresh_token": "test",
+            },
+        )
+        uuid, key2 = token.pk, token.key
+        response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(key2))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()['count'], 2)
+        self.assertEqual(
+            [item["alias"] for item in response.json()["results"]], ['', 'workflow-cli'])
+
+        response = self.client.delete(
+            '%s/%s' % (url, uuid), HTTP_AUTHORIZATION='Token {}'.format(key2))
+        self.assertEqual(response.status_code, 204)
+
+        response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(key1))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()['count'], 1)
+
+        response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(key2))
+        self.assertEqual(response.status_code, 401)
diff --git a/rootfs/api/tests/test_users.py b/rootfs/api/tests/test_users.py
deleted file mode 100644
index 1593f9b5c..000000000
--- a/rootfs/api/tests/test_users.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
-from api.tests import DryccTestCase
-
-User = get_user_model()
-
-
-class TestUsers(DryccTestCase):
-    """ Tests users endpoint"""
-
-    fixtures = ['tests.json']
-
-    def test_super_user_can_list(self):
-        user = User.objects.get(username='autotest')
-        token = Token.objects.get(user=user)
-
-        for url in ['/v2/users', '/v2/users/']:
-            response = self.client.get(url,
-                                       HTTP_AUTHORIZATION='token {}'.format(token))
-            self.assertEqual(response.status_code, 200, response.data)
-            self.assertEqual(len(response.data['results']), 4)
-
-    def test_enable(self):
-        user = User.objects.get(username='autotest')
-        token = Token.objects.get(user=user)
-        response = self.client.patch("/v2/users/autotest2/enable/",
-                                     HTTP_AUTHORIZATION='token {}'.format(token))
-        self.assertEqual(response.status_code, 204)
-        user = User.objects.get(username='autotest2')
-        self.assertEqual(user.is_active, True)
-
-    def test_disable(self):
-        user = User.objects.get(username='autotest')
-        token = Token.objects.get(user=user)
-        response = self.client.patch("/v2/users/autotest2/disable/",
-                                     HTTP_AUTHORIZATION='token {}'.format(token))
-        self.assertEqual(response.status_code, 204)
-        user = User.objects.get(username='autotest2')
-        self.assertEqual(user.is_active, False)
-
-    def test_non_super_user_cannot_list(self):
-        user = User.objects.get(username='autotest2')
-        token = Token.objects.get(user=user)
-
-        for url in ['/v2/users', '/v2/users/']:
-            response = self.client.get(url,
-                                       HTTP_AUTHORIZATION='token {}'.format(token))
-            self.assertEqual(response.status_code, 403)
diff --git a/rootfs/api/tests/test_volume.py b/rootfs/api/tests/test_volume.py
index fbf935cd0..20f908117 100644
--- a/rootfs/api/tests/test_volume.py
+++ b/rootfs/api/tests/test_volume.py
@@ -6,8 +6,6 @@
 """
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.conf import settings
-from rest_framework.authtoken.models import Token
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
 
@@ -22,19 +20,68 @@ class VolumeTest(DryccTransactionTestCase):
 
     def setUp(self):
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         self.app_id = self.create_app()
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
     def test_volumecreate(self, mock_requests):
         """Test that the serialized response contains only relevant data."""
         app_id = self.create_app()
+        # parameters is required
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': 'myvolume', 'size': '500G', 'type': 'nfs'
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+
+        # parameters format error
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': 'myvolume',
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+
+        # name regex error
+        name1 = 'myvolume_'
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': name1,
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name2 = '-myvolume'
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': name2,
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
+        name3 = 'my.volume'
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': name3,
+                'type': 'nfs',
+                'parameters': {'nfs': {'path': '/'}}
+            }
+        )
+        self.assertEqual(response.status_code, 400, response.data)
 
         response = self.client.post(
             '/v2/apps/{}/volumes'.format(app_id),
@@ -46,16 +93,15 @@ def test_volumecreate(self, mock_requests):
 
         for key in response.data:
             self.assertIn(key,
-                          ['uuid', 'owner', 'created', 'updated', 'app', 'name',
-                           'size', 'path'])
+                          ['uuid', 'created', 'updated', 'app', 'name',
+                           'size', 'path', 'type', 'parameters'])
 
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'name': 'myvolume',
             'size': '500G'
         }
-        self.assertDictContainsSubset(expected, response.data)
+        self.assertEqual(response.data, expected | response.data)
 
     def test_volume_list_unmount(self, mock_requests):
         """
@@ -84,8 +130,7 @@ def test_volume_expand(self, mock_requests):
         # create
         app_id = self.create_app()
         data = {'name': 'myvolume1', 'size': '500G'}
-        self.client.post('/v2/apps/{}/volumes'.format(app_id), data=data)
-
+        response = self.client.post('/v2/apps/{}/volumes'.format(app_id), data=data)
         # Patch
         url = '/v2/apps/{app_id}/volumes/{volume}'.format(app_id=app_id,
                                                           volume='myvolume1')
@@ -98,13 +143,31 @@ def test_volume_expand(self, mock_requests):
         url = '/v2/apps/{app_id}/volumes'.format(app_id=app_id)
         response = self.client.get(url)
         expected = {
-            'owner': self.user.username,
             'app': app_id,
             'name': 'myvolume1',
             'size': '1024G'
         }
         assert len(response.data["results"]) == 1
-        self.assertDictContainsSubset(expected, response.data["results"][0])
+        self.assertEqual(response.data["results"][0], expected | response.data["results"][0])
+        # nfs expand
+        response = self.client.post(
+            '/v2/apps/{}/volumes'.format(app_id),
+            data={
+                'name': 'myvolume2',
+                'type': 'nfs',
+                'parameters': {
+                    'nfs': {
+                        'server': 'test.drycc.cc',
+                        'path': '/',
+                    }
+                }
+            }
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        url = '/v2/apps/{app_id}/volumes/{volume}'.format(app_id=app_id,
+                                                          volume='myvolume2')
+        response = self.client.patch(url, {'name': 'myvolume2', 'size': '1024G'})
+        self.assertEqual(response.status_code, 400, response.data)
 
     def test_volume_delete(self, mock_requests):
         """
@@ -141,15 +204,19 @@ def test_volume_mount(self, mock_requests):
             {'name': 'myvolume1', 'size': '500G'}
         ]
         for _ in data:
-            self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+            response = self.client.post('/v2/apps/{}/volumes'.format(app_id), data=_)
+            self.assertEqual(response.status_code, 201, response.data)
 
         self.build_deploy(app_id)
         # mount
         url = '/v2/apps/{app_id}/volumes/myvolume1/path'.format(app_id=app_id)
         mount_path = {"path": {"web": "/data/web1"}}
         response = self.client.patch(url, data=mount_path)
-        expected = response.data['path']
-        self.assertEqual(mount_path['path'], expected)
+        expected = response.data['path']  # old data
+        self.assertEqual({}, expected)
+        url = '/v2/apps/{app_id}/volumes/myvolume1'.format(app_id=app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data["path"], mount_path["path"])
 
     def test_volume_unmount(self, mock_requests):
         # create
@@ -172,18 +239,26 @@ def test_volume_unmount(self, mock_requests):
         mount_path = {"path": {"web": "/data/web1"}}
         response = self.client.patch(url, data=mount_path)
         expected = response.data['path']
-        self.assertEqual(mount_path['path'], expected)
+        self.assertEqual({}, expected)
+        # check mount
+        url = '/v2/apps/{app_id}/volumes/myvolume1'.format(app_id=app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data["path"], mount_path["path"])
 
         # unmount
         url = '/v2/apps/{app_id}/volumes/myvolume1/path'.format(app_id=app_id)
         mount_path = {"path": {"web": None}}
         response = self.client.patch(url, data=mount_path)
         expected = response.data['path']
-        self.assertEqual({}, expected)
+        self.assertEqual(mount_path["path"], {'web': None})
+        # check mount
+        url = '/v2/apps/{app_id}/volumes/myvolume1'.format(app_id=app_id)
+        response = self.client.get(url)
+        self.assertEqual(response.data["path"], {})
 
     def build_deploy(self, app_id):
         # post a new build with procfile
-        url = "/v2/apps/{app_id}/builds".format(app_id=app_id)
+        url = "/v2/apps/{app_id}/build".format(app_id=app_id)
         body = {
             'image': 'autotest/example',
             'sha': 'a'*40,
@@ -201,7 +276,7 @@ def call_command(self, *args, **kwargs):
         from django.core.management import call_command
         out = StringIO()
         call_command(
-            "measure_volumes",
+            "upload_volume_usage",
             *args,
             stdout=out,
             stderr=StringIO(),
@@ -209,7 +284,7 @@ def call_command(self, *args, **kwargs):
         )
         return out.getvalue()
 
-    def test_measure_volumes(self, *args, **kwargs):
+    def test_upload_volume_usage(self, *args, **kwargs):
         # create
         app_id = self.create_app()
         data = [
diff --git a/rootfs/api/tests/test_workflow_manager.py b/rootfs/api/tests/test_workflow_manager.py
index aed92e8fb..c2dd93ebc 100644
--- a/rootfs/api/tests/test_workflow_manager.py
+++ b/rootfs/api/tests/test_workflow_manager.py
@@ -8,7 +8,6 @@
 from django.core.cache import cache
 from django.conf import settings
 from django.contrib.auth import get_user_model
-from rest_framework.authtoken.models import Token
 from api.tests import adapter, DryccTransactionTestCase
 import requests_mock
 
@@ -24,7 +23,7 @@ class ManagerTest(DryccTransactionTestCase):
     def setUp(self):
 
         self.user = User.objects.get(username='autotest')
-        self.token = Token.objects.get(user=self.user).key
+        self.token = self.get_or_create_token(self.user)
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
         self.app_id = self.create_app()
         self.user_id = 7
@@ -36,25 +35,23 @@ def setUp(self):
         self.client.credentials(HTTP_AUTHORIZATION='Token ' + token)
 
     def tearDown(self):
-        # Restore default tags to empty string
-        settings.DRYCC_DEFAULT_CONFIG_TAGS = ''
         # make sure every test has a clean slate for k8s mocking
         cache.clear()
 
     def test_block(self, mock_requests):
         response = self.client.post(
-            '/v2/manager/{}/{}/block/'.format("users", 7),
+            '/v2/manager/{}/{}/block/'.format("apps", self.app_id),
             data={'remark': 'Arrears blockade'},
         )
         self.assertEqual(response.status_code, 201)
 
     def test_unblock(self, mock_requests):
         response = self.client.post(
-            '/v2/manager/{}/{}/block/'.format("users", 7),
+            '/v2/manager/{}/{}/block/'.format("apps", self.app_id),
             data={'remark': 'Arrears blockade'},
         )
         self.assertEqual(response.status_code, 201)
         response = self.client.delete(
-            '/v2/manager/{}/{}/unblock/'.format("users", 7),
+            '/v2/manager/{}/{}/unblock/'.format("apps", self.app_id),
         )
         self.assertEqual(response.status_code, 204)
diff --git a/rootfs/api/tests/test_workspace.py b/rootfs/api/tests/test_workspace.py
new file mode 100644
index 000000000..39476bd52
--- /dev/null
+++ b/rootfs/api/tests/test_workspace.py
@@ -0,0 +1,196 @@
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from unittest import mock
+
+from api.models.workspace import Workspace, WorkspaceMember, WorkspaceInvitation
+from api.tests import DryccTestCase
+
+User = get_user_model()
+
+
+class WorkspaceTest(DryccTestCase):
+    """Tests creation and management of workspaces"""
+
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        self.user2 = User.objects.get(username='autotest2')
+        self.token2 = self.get_or_create_token(self.user2)
+
+    def tearDown(self):
+        cache.clear()
+
+    def test_workspace_lifecycle(self):
+        """Test workspace create, list, retrieve, update, delete"""
+        # Create
+        response = self.client.post(
+            '/v2/workspaces', {'name': 'testworkspace', 'email': 'test@example.com'}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        self.assertEqual(response.data['name'], 'testworkspace')
+
+        # Verify admin member created
+        workspace = Workspace.objects.get(name='testworkspace')
+        self.assertTrue(workspace.has_member(self.user, role='admin'))
+
+        # List
+        response = self.client.get('/v2/workspaces')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 1)
+
+        # Retrieve
+        response = self.client.get('/v2/workspaces/testworkspace')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['name'], 'testworkspace')
+
+        # Update (personal workspace NOT allowed)
+        response = self.client.patch(
+            f'/v2/workspaces/{self.user.username}', {'email': 'new@example.com'}
+        )
+        self.assertEqual(response.status_code, 404, response.data)
+
+        # Update
+        response = self.client.patch('/v2/workspaces/testworkspace', {'email': 'new@example.com'})
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['email'], 'new@example.com')
+
+        # Delete
+        response = self.client.delete('/v2/workspaces/testworkspace')
+        self.assertEqual(response.status_code, 204, response.data)
+
+    def test_workspace_isolation(self):
+        # User 1 creates workspace
+        self.client.post('/v2/workspaces', {'name': 'testisolated', 'email': 'test@example.com'})
+
+        # User 2 tries to access User 1's workspace
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token2)
+        response = self.client.get('/v2/workspaces/testisolated')
+        self.assertEqual(response.status_code, 404, response.data)  # Not found for user2
+
+
+class WorkspaceMemberTest(DryccTestCase):
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.user2 = User.objects.get(username='autotest2')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        # Create a workspace
+        self.client.post('/v2/workspaces', {'name': 'testmembers', 'email': 'test@example.com'})
+        self.workspace = Workspace.objects.get(name='testmembers')
+
+    def test_member_management(self):
+        # Add member via DB (since POST /members is not supported, users join via invitations)
+        WorkspaceMember.objects.create(user=self.user2, workspace=self.workspace, role='member')
+
+        # List members
+        response = self.client.get('/v2/workspaces/testmembers/members')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 2)
+
+        # Update member role (admin updating user2)
+        response = self.client.patch(
+            f'/v2/workspaces/testmembers/members/{self.user2.username}', {'role': 'viewer'}
+        )
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(response.data['role'], 'viewer')
+
+        # Non-admin user2 cannot update other members
+        token2 = self.get_or_create_token(self.user2)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + token2)
+        response = self.client.patch(
+            f'/v2/workspaces/testmembers/members/{self.user.username}', {'role': 'viewer'}
+        )
+        self.assertEqual(response.status_code, 403, response.data)
+
+        # Admin delete member
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+        response = self.client.delete(f'/v2/workspaces/testmembers/members/{self.user2.username}')
+        self.assertEqual(response.status_code, 204, response.data)
+
+
+class WorkspaceInvitationTest(DryccTestCase):
+    fixtures = ['tests.json']
+
+    def setUp(self):
+        self.user = User.objects.get(username='autotest')
+        self.user2 = User.objects.get(username='autotest2')
+        self.token = self.get_or_create_token(self.user)
+        self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
+
+        self.client.post('/v2/workspaces', {'name': 'testinvite', 'email': 'test@example.com'})
+        self.workspace = Workspace.objects.get(name='testinvite')
+
+    @mock.patch('api.models.workspace.send_mail')
+    def test_invitation_lifecycle(self, mock_send_mail):
+        # Create invitation
+        response = self.client.post(
+            '/v2/workspaces/testinvite/invitations', {'email': self.user2.email}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        mock_send_mail.assert_called_once()
+
+        invitation = WorkspaceInvitation.objects.get(email=self.user2.email)
+
+        # List invitations
+        response = self.client.get('/v2/workspaces/testinvite/invitations')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertEqual(len(response.data['results']), 1)
+
+        # Accept invitation via API (JSON response)
+        response = self.client.get(f'/v2/workspaces/testinvite/invitations/{invitation.token}')
+        self.assertEqual(response.status_code, 200, response.data)
+        self.assertIn('workspace_name', response.data)
+        self.assertEqual(response.data['workspace_name'], 'testinvite')
+        self.assertIn('user_exists', response.data)
+        self.assertIn('register_url', response.data)
+
+        # Verify user became a member
+        self.assertTrue(
+            WorkspaceMember.objects.filter(
+                workspace=self.workspace,
+                user=self.user2
+            ).exists()
+        )
+
+        # Test delete invitation
+        response = self.client.post(
+            '/v2/workspaces/testinvite/invitations', {'email': 'test-invite2@example.com'}
+        )
+        invitation2 = WorkspaceInvitation.objects.get(email='test-invite2@example.com')
+        response = self.client.delete(
+            f'/v2/workspaces/testinvite/invitations/{invitation2.token}'
+        )
+        self.assertEqual(response.status_code, 204, response.data)
+
+    @mock.patch('api.models.workspace.send_mail')
+    def test_invitation_accept_browser(self, mock_send_mail):
+        """Test accepting invitation via browser returns HTML"""
+        response = self.client.post(
+            '/v2/workspaces/testinvite/invitations', {'email': self.user2.email}
+        )
+        self.assertEqual(response.status_code, 201, response.data)
+        invitation = WorkspaceInvitation.objects.get(email=self.user2.email)
+
+        # Accept invitation via browser (HTML response)
+        response = self.client.get(
+            f'/v2/workspaces/testinvite/invitations/{invitation.token}',
+            HTTP_ACCEPT='text/html',
+        )
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response['Content-Type'], 'text/html; charset=utf-8')
+        self.assertIn(b'testinvite', response.content)
+
+        # Verify user became a member
+        self.assertTrue(
+            WorkspaceMember.objects.filter(
+                workspace=self.workspace,
+                user=self.user2
+            ).exists()
+        )
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
index 9eeef8753..bf4baba6b 100644
--- a/rootfs/api/urls.py
+++ b/rootfs/api/urls.py
@@ -15,47 +15,96 @@
 app_urlpatterns = [
     re_path(r'^', include(router.urls)),
     re_path(r'auth/login/?$', views.AuthLoginView.as_view({"post": "login"})),
+    re_path(r'auth/token/?$', views.AuthTokenView.as_view({"post": "token"})),
     re_path(r'auth/token/(?P<key>[-_\w]+)/?$', views.AuthTokenView.as_view({"get": "token"})),
+    # Workspaces management URLs (keep workspaces prefix, no user prefix)
+    re_path(r'^workspaces/?$',
+            views.WorkspaceViewSet.as_view({'get': 'list', 'post': 'create'}),
+            name='workspace_list'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/?$',
+            views.WorkspaceViewSet.as_view(
+                {'get': 'retrieve', 'patch': 'partial_update', 'delete': 'destroy'}),
+            name='workspace_detail'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/members/?$',
+            views.WorkspaceMemberViewSet.as_view({'get': 'list'}),
+            name='workspace_member_list'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/members/(?P<user>[-_\w]+)/?$',
+            views.WorkspaceMemberViewSet.as_view(
+                {'get': 'retrieve', 'patch': 'partial_update', 'delete': 'destroy'}),
+            name='workspace_member_detail'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/invitations/?$',
+            views.WorkspaceInvitationViewSet.as_view({'get': 'list', 'post': 'create'}),
+            name='workspace_invitation_list'),
+    re_path(r'^workspaces/(?P<name>[-_\w]+)/invitations/(?P<uid>[-_\w]+)/?$',
+            views.WorkspaceInvitationViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'}),
+            name='workspace_invitation_detail'),
+    # limits
+    re_path(
+        r'^limits/specs/?$',
+        views.LimitSpecViewSet.as_view({'get': 'list'})),
+    re_path(
+        r'^limits/plans/?$',
+        views.LimitPlanViewSet.as_view({'get': 'list'})),
+    re_path(
+        r'^limits/plans/(?P<id>[-.\w]+)/?$',
+        views.LimitPlanViewSet.as_view({'get': 'retrieve'})),
     # application release components
     re_path(
-        r"^apps/(?P<id>{})/config/?$".format(settings.APP_URL_REGEX),
-        views.ConfigViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
-    re_path(
-        r"^apps/(?P<id>{})/builds/(?P<uuid>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
-        views.BuildViewSet.as_view({'get': 'retrieve'})),
+        r"^apps/(?P<id>{})/build/?$".format(settings.APP_URL_REGEX),
+        views.BuildViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
     re_path(
-        r"^apps/(?P<id>{})/builds/?$".format(settings.APP_URL_REGEX),
-        views.BuildViewSet.as_view({'get': 'list', 'post': 'create'})),
+        r"^apps/(?P<id>{})/config/?$".format(settings.APP_URL_REGEX),
+        views.ConfigViewSet.as_view({'get': 'retrieve', 'post': 'create', 'delete': 'destroy'})),
     re_path(
         r"^apps/(?P<id>{})/releases/v(?P<version>[0-9]+)/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'get': 'retrieve'})),
+    re_path(
+        r"^apps/(?P<id>{})/releases/deploy/?$".format(settings.APP_URL_REGEX),
+        views.ReleaseViewSet.as_view({'post': 'deploy'})),
     re_path(
         r"^apps/(?P<id>{})/releases/rollback/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'post': 'rollback'})),
     re_path(
         r"^apps/(?P<id>{})/releases/?$".format(settings.APP_URL_REGEX),
         views.ReleaseViewSet.as_view({'get': 'list'})),
-    # restart pods
+    # list/delete pods
     re_path(
-        r"^apps/(?P<id>{})/pods/restart/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'post': 'restart'})),
+        r"^apps/(?P<id>{})/pods/?$".format(settings.APP_URL_REGEX),
+        views.PodViewSet.as_view({'get': 'list', 'delete': 'destroy'})),
+    # describe pod
     re_path(
-        r"^apps/(?P<id>{})/pods/(?P<type>[-_\w.]+)/restart/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'post': 'restart'})),
-    # list pods
+        r"^apps/(?P<id>{})/pods/(?P<name>{})/describe/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
+        views.PodViewSet.as_view({'get': 'describe'})),
+    # restart deployment/ptype's pods
     re_path(
-        r"^apps/(?P<id>{})/pods/(?P<type>[-_\w]+)/(?P<name>[-_\w]+)/?$".format(
-            settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list'})),
+        r"^apps/(?P<id>{})/ptypes/restart/?$".format(settings.APP_URL_REGEX),
+        views.PtypeViewSet.as_view({'post': 'restart'})),
+    # clean old k8s resource
     re_path(
-        r"^apps/(?P<id>{})/pods/(?P<type>[-_\w.]+)/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list'})),
+        r"^apps/(?P<id>{})/ptypes/clean/?$".format(settings.APP_URL_REGEX),
+        views.PtypeViewSet.as_view({'post': 'clean'})),
+    # scale ptype replcas
     re_path(
-        r"^apps/(?P<id>{})/pods/?$".format(settings.APP_URL_REGEX),
-        views.PodViewSet.as_view({'get': 'list'})),
+        r"^apps/(?P<id>{})/ptypes/scale/?$".format(settings.APP_URL_REGEX),
+        views.PtypeViewSet.as_view({'post': 'scale'})),
+    # list ptypes
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/?$".format(settings.APP_URL_REGEX),
+        views.PtypeViewSet.as_view({'get': 'list'})),
+    # describe ptypes
+    re_path(
+        r"^apps/(?P<id>{})/ptypes/(?P<name>{})/describe/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
+        views.PtypeViewSet.as_view({'get': 'describe'})),
+    # list events
+    re_path(
+        r"^apps/(?P<id>{})/events/?$".format(settings.APP_URL_REGEX),
+        views.EventViewSet.as_view({'get': 'list'})),
     # application domains
     re_path(
-        r"^apps/(?P<id>{})/domains/(?P<domain>\**\.?[-\._\w]+)/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/domains/(?P<domain>{})/?$".format(
+            settings.APP_URL_REGEX, settings.DOMAIN_URL_REGEX),
         views.DomainViewSet.as_view({'delete': 'destroy'})),
     re_path(
         r"^apps/(?P<id>{})/domains/?$".format(settings.APP_URL_REGEX),
@@ -64,46 +113,27 @@
     re_path(
         r"^apps/(?P<id>{})/services/?$".format(settings.APP_URL_REGEX),
         views.ServiceViewSet.as_view({'post': 'create_or_update',
-                                     'get': 'list', 'delete': 'delete'})),
-    # application actions
-    re_path(
-        r"^apps/(?P<id>{})/scale/?$".format(settings.APP_URL_REGEX),
-        views.AppViewSet.as_view({'post': 'scale'})),
-    re_path(
-        r"^apps/(?P<id>{})/run/?$".format(settings.APP_URL_REGEX),
-        views.AppViewSet.as_view({'post': 'run'})),
+                                     'get': 'list', 'delete': 'destroy'})),
     # application settings
     re_path(
         r"^apps/(?P<id>{})/settings/?$".format(settings.APP_URL_REGEX),
-        views.AppSettingsViewSet.as_view(
-            {'get': 'retrieve', 'post': 'create', 'delete': 'remove'})),
-    # application canary
-    re_path(
-        r"^apps/(?P<id>{})/canary/release/?$".format(settings.APP_URL_REGEX),
-        views.CanaryViewSet.as_view({'post': 'release'})),
-    re_path(
-        r"^apps/(?P<id>{})/canary/rollback/?$".format(settings.APP_URL_REGEX),
-        views.CanaryViewSet.as_view({'post': 'rollback'})),
+        views.AppSettingsViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
     # application TLS settings
     re_path(
         r"^apps/(?P<id>{})/tls/?$".format(settings.APP_URL_REGEX),
         views.TLSViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
-    # apps sharing
-    re_path(
-        r"^apps/(?P<id>{})/perms/(?P<username>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
-        views.AppPermsViewSet.as_view({'delete': 'destroy'})),
-    re_path(
-        r"^apps/(?P<id>{})/perms/?$".format(settings.APP_URL_REGEX),
-        views.AppPermsViewSet.as_view({'get': 'list', 'post': 'create'})),
     # application volumes
     re_path(
         r"^apps/(?P<id>{})/volumes/?$".format(settings.APP_URL_REGEX),
         views.AppVolumesViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
-        views.AppVolumesViewSet.as_view({'patch': 'expand', 'delete': 'destroy'})),
+        r"^apps/(?P<id>{})/volumes/(?P<name>{})/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
+        views.AppVolumesViewSet.as_view(
+            {'get': 'retrieve', 'patch': 'expand', 'delete': 'destroy'})),
     re_path(
-        r"^apps/(?P<id>{})/volumes/(?P<name>[-_\w]+)/path/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/volumes/(?P<name>{})/path/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppVolumesViewSet.as_view({'patch': 'path'})),
     # application resources
     re_path(r"^resources/services/?$", views.AppResourcesViewSet.as_view({'get': 'services'})),
@@ -114,19 +144,34 @@
         r"^apps/(?P<id>{})/resources/?$".format(settings.APP_URL_REGEX),
         views.AppResourcesViewSet.as_view({'get': 'list', 'post': 'create'})),
     re_path(
-        r"^apps/(?P<id>{})/resources/(?P<name>[-_\w]+)/?$".format(settings.APP_URL_REGEX),
-        views.AppSingleResourceViewSet.as_view({
-            'get': 'retrieve',
-            'delete': 'destroy',
-            'put': 'update'
-        })),
+        r"^apps/(?P<id>{})/resources/(?P<name>{})/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
+        views.AppSingleResourceViewSet.as_view(
+            {'get': 'retrieve', 'delete': 'destroy', 'put': 'update'})),
     re_path(
-        r"^apps/(?P<id>{})/resources/(?P<name>[-_\w]+)/binding/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/resources/(?P<name>{})/binding/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.AppResourceBindingViewSet.as_view({'patch': 'binding'})),
+    # certificates
+    re_path(
+        r'^apps/(?P<id>{})/certs/(?P<name>{})/domain/(?P<domain>{})?/?$'.format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX, settings.DOMAIN_URL_REGEX),
+        views.CertificateViewSet.as_view({'delete': 'detach', 'post': 'attach'})),
+    re_path(
+        r'^apps/(?P<id>{})/certs/(?P<name>{})/?$'.format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
+        views.CertificateViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
+    re_path(
+        r'^apps/(?P<id>{})/certs/?$'.format(settings.APP_URL_REGEX),
+        views.CertificateViewSet.as_view({'get': 'list', 'post': 'create'})),
+    # application actions
+    re_path(
+        r"^apps/(?P<id>{})/run/?$".format(settings.APP_URL_REGEX),
+        views.AppViewSet.as_view({'post': 'run'})),
     # apps base endpoint
     re_path(
         r"^apps/(?P<id>{})/?$".format(settings.APP_URL_REGEX),
-        views.AppViewSet.as_view({'get': 'retrieve', 'post': 'update', 'delete': 'destroy'})),
+        views.AppViewSet.as_view({'get': 'retrieve', 'patch': 'transfer', 'delete': 'destroy'})),
     re_path(
         r'^apps/?$',
         views.AppViewSet.as_view({'get': 'list', 'post': 'create'})),
@@ -139,7 +184,7 @@
         views.KeyViewSet.as_view({'get': 'list', 'post': 'create'})),
     # hooks
     re_path(
-        r'^hooks/keys/(?P<id>{})/(?P<username>[-_\w]+)/?$'.format(settings.APP_URL_REGEX),
+        r'^hooks/keys/(?P<id>{})/(?P<username>[\w.@+-]+)/?$'.format(settings.APP_URL_REGEX),
         views.KeyHookViewSet.as_view({'get': 'users'})),
     re_path(
         r'^hooks/keys/(?P<id>{})/?$'.format(settings.APP_URL_REGEX),
@@ -156,58 +201,35 @@
     # authn / authz
     re_path(
         r'^auth/whoami/?$',
-        views.UserManagementViewSet.as_view({'get': 'list'})),
-    # admin sharing
-    re_path(
-        r'^admin/perms/(?P<username>[\w.@+-]+)/?$',
-        views.AdminPermsViewSet.as_view({'delete': 'destroy'})),
-    re_path(
-        r'^admin/perms/?$',
-        views.AdminPermsViewSet.as_view({'get': 'list', 'post': 'create'})),
-    # certificates
-    re_path(
-        r'^certs/(?P<name>[-_*.\w]+)/domain/(?P<domain>\**\.?[-\._\w]+)?/?$',
-        views.CertificateViewSet.as_view({'delete': 'detach', 'post': 'attach'})),
-    re_path(
-        r'^certs/(?P<name>[-_*.\w]+)/?$',
-        views.CertificateViewSet.as_view({
-            'get': 'retrieve',
-            'delete': 'destroy'
-        })),
-    re_path(
-        r'^certs/?$',
-        views.CertificateViewSet.as_view({'get': 'list', 'post': 'create'})),
+        views.UserManagementViewSet.as_view({'get': 'whoami'})),
     # gateways
     re_path(
         r"^apps/(?P<id>{})/gateways/?$".format(settings.APP_URL_REGEX),
         views.GatewayViewSet.as_view(
-            {'post': 'create_or_update', 'get': 'list', 'delete': 'delete'})),
+            {'post': 'create_or_update', 'get': 'list', 'delete': 'destroy'})),
     # routes
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)?/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})?/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view(
-            {'post': 'create', 'get': 'list', 'delete': 'delete'})),
+            {'post': 'create', 'get': 'list', 'delete': 'destroy'})),
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)/attach/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})/attach/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'patch': 'attach'})),
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)/detach/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})/detach/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'patch': 'detach'})),
     re_path(
-        r"^apps/(?P<id>{})/routes/(?P<name>[-_\w]+)/rules/?$".format(settings.APP_URL_REGEX),
+        r"^apps/(?P<id>{})/routes/(?P<name>{})/rules/?$".format(
+            settings.APP_URL_REGEX, settings.NAME_REGEX),
         views.RouteViewSet.as_view({'get': 'get', 'put': 'set'})),
-    # users
-    re_path(
-        r'^users/?$',
-        views.UserView.as_view({'get': 'list'})),
-    re_path(
-        r'^users/(?P<username>[\w.@+-]+)/enable/?$',
-        views.UserView.as_view({'patch': 'enable'})),
     re_path(
-        r'^users/(?P<username>[\w.@+-]+)/disable/?$',
-        views.UserView.as_view({'patch': 'disable'})),
+        r'^apps/(?P<id>{})/metrics/?$'.format(settings.APP_URL_REGEX),
+        views.MetricView.as_view({'get': 'metric'})),
     re_path(
-        r'^apps/(?P<id>{})/metrics/(?P<type>[a-z0-9]+(\-[a-z0-9]+)*)/status/?$'.format(
+        r'^apps/(?P<id>{})/metrics/(?P<ptype>[a-z0-9]+(\-[a-z0-9]+)*)/status/?$'.format(
             settings.APP_URL_REGEX),
         views.MetricView.as_view({'get': 'status'})),
     re_path(
@@ -216,21 +238,41 @@
     re_path(
         r'^manager/(?P<type>[\w.@+-]+)s/(?P<id>{})/unblock/?$'.format(settings.APP_URL_REGEX),
         views.WorkflowManagerViewset.as_view({'delete': 'unblock'})),
-    # social login is placed at the end of the URL match
-    re_path(r'^login/(?P<backend>[^/]+){0}$'.format(extra), views.auth, name='begin'),
-    re_path(r'^complete/(?P<backend>[^/]+){0}$'.format(extra), views.complete, name='complete'),
-    re_path('', include('social_django.urls', namespace='social')),
+    # quickwit
+    re_path(
+        r'^quickwit/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$', views.QuickwitProxyView.as_view()),
+    # prometheus
+    re_path(
+        r'^prometheus/(?P<username>[\w.@+-]+)/(?P<path>.+)/?$',
+        views.PrometheusProxyView.as_view()),
+    # tokens
+    re_path(r'^tokens/?$', views.TokenViewSet.as_view({'get': 'list'})),
+    re_path(
+        r"^tokens/(?P<pk>[-_\w]+)/?$",
+        views.TokenViewSet.as_view({'get': 'retrieve', 'delete': 'destroy'})),
 ]
 
-webhook_urlpatterns = [
+metric_urlpatterns = [
+    re_path(r'^metrics/?$', views.MetricsProxyView.as_view()),
+]
+
+mutate_urlpatterns = [
     re_path(
-        r'^webhooks/scale/(?P<token>.+)/?$',
-        views.AdmissionWebhookViewSet.as_view({'post': 'scale'})
+        r'^mutate/(?P<key>.+)/?$',
+        views.AdmissionWebhookViewSet.as_view({'post': 'handle'})
     ),
 ]
 
-# If there is a mutating admission webhook configuration, use webhook url
-if settings.DRYCC_ADMISSION_WEBHOOK_TOKEN:
-    urlpatterns = webhook_urlpatterns
+# social login is placed at the end of the URL match
+social_urlpatterns = [
+    re_path(r'^login/(?P<backend>[^/]+){0}$'.format(extra), views.auth, name='begin'),
+    re_path(r'^complete/(?P<backend>[^/]+){0}$'.format(extra), views.complete, name='complete'),
+    re_path('', include('social_django.urls', namespace='social')),
+]
+
+if settings.RUNNER == 'metric':
+    urlpatterns = metric_urlpatterns
+elif settings.RUNNER == 'mutate':
+    urlpatterns = mutate_urlpatterns
 else:
-    urlpatterns = app_urlpatterns
+    urlpatterns = app_urlpatterns + social_urlpatterns
diff --git a/rootfs/api/utils.py b/rootfs/api/utils.py
index da65f1ab7..d2aa1d519 100644
--- a/rootfs/api/utils.py
+++ b/rootfs/api/utils.py
@@ -3,6 +3,8 @@
 """
 import os
 import re
+import uuid
+import time
 import base64
 import string
 import concurrent
@@ -13,31 +15,41 @@
 import pkgutil
 import inspect
 import requests
+import importlib
+import jsonschema
 from copy import deepcopy
+from urllib.parse import urljoin
 from django.db import models
+from django.conf import settings
+from django.core.cache import cache
+from django.utils import timezone
+from asgiref.sync import sync_to_async
 from requests_toolbelt import user_agent
 from api import __version__ as drycc_version
 from rest_framework.exceptions import ValidationError
 
+
 logger = logging.getLogger(__name__)
 
 
-session = None
+def get_httpclient():
+    session = requests.Session()
+    session.headers = {
+        # https://toolbelt.readthedocs.org/en/latest/user-agent.html#user-agent-constructor
+        'User-Agent': user_agent('Drycc Controller', drycc_version),
+    }
+    # `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
+    # http://docs.python-requests.org/en/latest/api/#requests.adapters.HTTPAdapter
+    session.mount('http://', requests.adapters.HTTPAdapter(max_retries=10))
+    session.mount('https://', requests.adapters.HTTPAdapter(max_retries=10))
+    return session
 
 
-def get_session():
-    global session
-    if session is None:
-        session = requests.Session()
-        session.headers = {
-            # https://toolbelt.readthedocs.org/en/latest/user-agent.html#user-agent-constructor
-            'User-Agent': user_agent('Drycc Controller', drycc_version),
-        }
-        # `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
-        # http://docs.python-requests.org/en/latest/api/#requests.adapters.HTTPAdapter
-        session.mount('http://', requests.adapters.HTTPAdapter(max_retries=10))
-        session.mount('https://', requests.adapters.HTTPAdapter(max_retries=10))
-    return session
+def get_scheduler(metadata=None):
+    mod = importlib.import_module(settings.SCHEDULER_MODULE)
+    metadata = metadata if metadata is not None else {}
+    metadata = dict_merge(metadata, {"annotations": {"app.kubernetes.io/managed-by": "drycc"}})
+    return mod.SchedulerClient(settings.SCHEDULER_URL, settings.K8S_API_VERIFY_TLS, metadata)
 
 
 def import_all_models():
@@ -60,6 +72,14 @@ def validate_label(value):
     match = re.match(r'^[a-z0-9-]+$', value)
     if not match:
         raise ValidationError("Can only contain a-z (lowercase), 0-9 and hyphens")
+    validate_reserved_names(value)
+
+
+def validate_reserved_names(value):
+    """A value cannot use some reserved names."""
+    for reserved_name_pattern in settings.RESERVED_NAME_PATTERNS:
+        if re.match(reserved_name_pattern, value):
+            raise ValidationError('{} is a reserved name.'.format(value))
 
 
 def random_string(num):
@@ -72,6 +92,30 @@ def generate_app_name():
     return "{}-{}".format(random_string(6), random_string(8))
 
 
+def send_app_log(app_id, msg, level=logging.INFO):
+    if not settings.QUICKWIT_INDEXER_URL:
+        return
+    pod_ip = os.environ.get("POD_IP", "unknown")
+    pod_name = os.environ.get("POD_NAME", "unknown")
+    namespace = os.environ.get("WORKFLOW_NAMESPACE", "unknown")
+    docker_id = hashlib.sha256(f"{namespace}:{pod_name}:{pod_ip}".encode("utf-8")).hexdigest()
+    data = {
+        "kubernetes": {
+            "container_name": "drycc-controller",
+            "docker_id": docker_id,
+            "namespace_name": app_id,
+            "pod_name": pod_name,
+        },
+        "log": f"{logging.getLevelName(level)}\t{app_id}\t{msg}",
+        "offset": int(time.time()),
+        "stream": "stdout",
+        "timestamp": timezone.now().isoformat(),
+    }
+    index = f"{settings.QUICKWIT_LOG_INDEX_PREFIX}{app_id}"
+    with get_httpclient() as session:
+        session.post(urljoin(settings.QUICKWIT_INDEXER_URL, f"/api/v1/{index}/ingest"), json=data)
+
+
 def dict_diff(dict1, dict2):
     """
     Returns the added, changed, and deleted items in dict1 compared with dict2.
@@ -153,8 +197,9 @@ def apply_tasks(tasks):
     if not tasks:
         return
 
-    executor = concurrent.futures.ThreadPoolExecutor(5)
-    for future in [executor.submit(task) for task in tasks]:
+    executor = concurrent.futures.ThreadPoolExecutor(settings.DRYCC_APPLY_TASKS)
+    for future, callback in [(executor.submit(task[0]), task[1]) for task in tasks]:
+        future.add_done_callback(callback)
         error = future.exception()
         if error is not None:
             raise error
@@ -186,14 +231,103 @@ def unit_to_bytes(size):
     return round(_)
 
 
-def unit_to_millicpu(cpu):
-    cpu = cpu.lower()
-    if cpu.endswith("m") and cpu[:-1].isdigit():
-        return int(cpu[:-1])
-    elif cpu.isdigit():
-        return int(cpu) * 1000
-    else:
-        raise ValueError("Unrecognized CPU unit: %s" % cpu)
+def validate_json(value, schema, raise_exception=jsonschema.ValidationError):
+    if value is not None:
+        try:
+            jsonschema.validate(value, schema)
+        except jsonschema.ValidationError as e:
+            raise raise_exception("could not validate {}: {}".format(value, e.message))
+    return value
+
+
+def get_local_host(request):
+    uri = request.build_absolute_uri()
+    return uri[0:uri.find(request.path)]
+
+
+class CacheLock(object):
+
+    def __init__(self, key):
+        self.key = key
+        self.value = uuid.uuid4().hex
+
+    def acquire(self, blocking=True, timeout=120):
+        value = None
+        for _ in range(timeout):
+            value = cache.get_or_set(self.key, self.value, timeout)
+            if not blocking or value == self.value:
+                break
+            time.sleep(1)
+        return value == self.value
+
+    def release(self):
+        value = cache.get(self.key, None)
+        if value != self.value:
+            return
+        cache.delete(self.key)
+
+
+class DeployLock(CacheLock):
+
+    def __init__(self, app_id):
+        self.ptypes_key = f"app:deploy:procfile:types:{app_id}"
+        super(DeployLock, self).__init__(f"app:deploy:lock:{app_id}")
+
+    def locked(self, ptypes):
+        value = cache.get(self.ptypes_key, [])
+        locked_ptypes = []
+        for ptype in ptypes:
+            if ptype in value:
+                locked_ptypes.append(ptype)
+        return locked_ptypes
+
+    def acquire(self, ptypes, force=False):
+        try:
+            if super(DeployLock, self).acquire():
+                value = cache.get(self.ptypes_key, [])
+                if not force and len(self.locked(ptypes)) > 0:
+                    return False
+                value.extend(ptypes)
+                value = list(set(value))
+                cache.set(self.ptypes_key, value, timeout=3600)
+                return True
+        finally:
+            super(DeployLock, self).release()
+        return False
+
+    def release(self, ptypes):
+        try:
+            if super(DeployLock, self).acquire():
+                value = cache.get(self.ptypes_key, [])
+                if value is None:
+                    return
+                for ptype in ptypes:
+                    if ptype in value:
+                        value.remove(ptype)
+                cache.set(self.ptypes_key, value)
+        finally:
+            super(DeployLock, self).release()
+
+
+class SyncIterToAsyncIter(object):
+
+    def __init__(self, iter_content):
+        self.iter_content = iter_content
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        @sync_to_async
+        def async_next():
+            try:
+                return next(self.iter_content)
+            except StopIteration:
+                raise StopAsyncIteration
+        return await async_next()
+
+
+iter_to_aiter = SyncIterToAsyncIter
 
 
 if __name__ == "__main__":
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
index de5a082bd..131889d5a 100644
--- a/rootfs/api/views.py
+++ b/rootfs/api/views.py
@@ -1,38 +1,59 @@
 """
 RESTful view classes for presenting Drycc API objects.
 """
+import re
 import uuid
 import logging
 import json
+import ssl
+import time
+import zlib
+import random
+import secrets
+import aiohttp
+import requests
+import warnings
+
+from urllib.parse import urljoin
+from django.db import transaction
+from django.db.models import Q
 from django.core.cache import cache
 from django.http import Http404, HttpResponse
 from django.conf import settings
 from django.contrib.auth import get_user_model
-from django.shortcuts import get_object_or_404, redirect
-from guardian.shortcuts import assign_perm, get_objects_for_user, \
-    get_users_with_perms, remove_perm
+from django.shortcuts import get_object_or_404, redirect, render
 from django.views.generic import View
-from rest_framework import renderers, status, filters
-from rest_framework.exceptions import PermissionDenied, NotFound
+from django.utils.decorators import method_decorator
+from django.views.decorators.cache import cache_page
+from django.views.decorators.vary import vary_on_headers
+from rest_framework import status, filters
 from rest_framework.permissions import IsAuthenticated, AllowAny
 from rest_framework.response import Response
-from rest_framework.viewsets import GenericViewSet
+from rest_framework.viewsets import GenericViewSet, ModelViewSet
+from rest_framework.exceptions import ValidationError
+from rest_framework.renderers import JSONRenderer, TemplateHTMLRenderer
 
-from api import monitor, models, permissions, serializers, viewsets, authentication
-from api.tasks import scale_app, restart_app, mount_app, downstream_model_owner
-from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException, \
-    UnprocessableEntity
+from api import monitor, models, permissions, serializers, viewsets, authentication, __version__
+from api.tasks import scale_app, restart_app, mount_app, delete_pod, scale_resources
+from api.exceptions import AlreadyExists, ServiceUnavailable, DryccException
 
 from django.views.decorators.cache import never_cache
 from django.contrib.auth import REDIRECT_FIELD_NAME
 from django.views.decorators.csrf import csrf_exempt
+from django.http.response import JsonResponse, StreamingHttpResponse
+from channels.db import database_sync_to_async
 from social_django.utils import psa
 from social_django.views import _do_login
 from social_core.utils import setting_name
+from api import admissions, utils
+from api.apps_extra.social_core.backends import OauthCacheManager
 from api.apps_extra.social_core.actions import do_auth, do_complete
 
+
 User = get_user_model()
 logger = logging.getLogger(__name__)
+is_loopback = re.compile(r'^(localhost|127\.0\.0\.1)(:\d+)?/').match
+oauth_cache_manager = OauthCacheManager()
 NAMESPACE = getattr(settings, setting_name('URL_NAMESPACE'), None) or 'social'
 
 
@@ -83,13 +104,42 @@ def complete(request, backend, *args, **kwargs):
 class AuthLoginView(GenericViewSet):
 
     permission_classes = (AllowAny, )
+    serializer_class = serializers.AuthSerializer
 
     def login(self, request, *args, **kwargs):
-        def get_local_host(request):
-            uri = request.build_absolute_uri()
-            return uri[0:uri.find(request.path)]
-        res = redirect(get_local_host(request) + "/v2/login/drycc/?key=" + uuid.uuid4().hex)
-        return res
+        key = uuid.uuid4().hex
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        username = serializer.validated_data.get('username')
+        password = serializer.validated_data.get('password')
+        if username and password:
+            return self._create_interactive_response(username, password, key)
+        return self._create_browser_response(key)
+
+    def _create_browser_response(self, key):
+        uri = self.request.build_absolute_uri()
+        return redirect(f"{uri[0:uri.find(self.request.path)]}/v2/login/drycc/?key={key}")
+
+    def _create_interactive_response(self, username, password, key):
+        # Get token endpoint from OIDC discovery
+        token_url = oauth_cache_manager.drycc_oauth.access_token_url()
+        client_id, client_secret = oauth_cache_manager.drycc_oauth.get_key_and_secret()
+        response = requests.post(
+            token_url,
+            data={
+                'grant_type': 'password',
+                'client_id': client_id,
+                'client_secret': client_secret,
+                'username': username,
+                'password': password,
+            },
+        )
+        if response.status_code != 200:
+            raise DryccException(response.content)
+        state = uuid.uuid4().hex
+        oauth_cache_manager.set_state(key, state)
+        oauth_cache_manager.set_token(state, response.json())
+        return HttpResponse(json.dumps({"key": key}))
 
 
 class AuthTokenView(GenericViewSet):
@@ -97,24 +147,28 @@ class AuthTokenView(GenericViewSet):
     permission_classes = (AllowAny, )
 
     def token(self, request, *args, **kwargs):
-        state = cache.get("oidc_key_" + self.kwargs['key'], "")
-        token = cache.get("oidc_state_" + state, {})
-        if not token.get('token'):
-            return HttpResponse(status=404)
-        return HttpResponse(json.dumps(token))
+        if 'key' in self.kwargs:
+            oauth = oauth_cache_manager.get_token(self.kwargs['key'])
+        else:
+            try:
+                oauth = json.loads(request.body.decode("utf8"))
+            except json.decoder.JSONDecodeError:
+                return HttpResponse(status=400)
+        if oauth and 'access_token' in oauth:
+            user = oauth_cache_manager.get_user(oauth['access_token'])
+            alias = request.query_params.get('alias', '')
+            token = models.base.Token(owner=user, alias=alias, oauth=oauth)
+            token.save()
+            return HttpResponse(json.dumps(
+                {"uuid": str(token.uuid), "token": token.key, "username": user.username}))
+        return HttpResponse(status=404)
 
 
 class UserManagementViewSet(GenericViewSet):
     serializer_class = serializers.UserSerializer
 
-    def get_queryset(self):
-        return User.objects.filter(pk=self.request.user.pk)
-
-    def get_object(self):
-        return self.get_queryset()[0]
-
-    def list(self, request, **kwargs):
-        user = self.get_object()
+    def whoami(self, request, **kwargs):
+        user = get_object_or_404(User, pk=self.request.user.pk)
         serializer = self.get_serializer(user, many=False)
         return Response(serializer.data)
 
@@ -125,13 +179,17 @@ class WorkflowManagerViewset(GenericViewSet):
 
     def block(self, request,  **kwargs):
         try:
-            blocklist, _ = models.blocklist.Blocklist.objects.get_or_create(
+            blocklist, created = models.blocklist.Blocklist.objects.get_or_create(
                 id=kwargs['id'],
                 type=models.blocklist.Blocklist.get_type(kwargs["type"]),
                 defaults={"remark": request.data.get("remark")}
             )
             for app in blocklist.related_apps:
-                scale_app.delay(app, app.owner, {key: 0 for key in app.structure.keys()})
+                if created:
+                    app.suspended_state = blocklist.suspended_state(app)
+                    app.save()
+                # scale to 0
+                scale_resources.delay(blocklist, app, app.suspended_state, "block")
             return HttpResponse(status=201)
         except ValueError as e:
             logger.info(e)
@@ -139,10 +197,17 @@ def block(self, request,  **kwargs):
 
     def unblock(self, request,  **kwargs):
         try:
-            models.blocklist.Blocklist.objects.filter(
+            blocklists = models.blocklist.Blocklist.objects.filter(
                 id=kwargs['id'],
                 type=models.blocklist.Blocklist.get_type(kwargs["type"])
-            ).delete()
+            )
+            for blocklist in blocklists:
+                for app in blocklist.related_apps:
+                    # scale to up
+                    scale_resources.delay(blocklist, app, app.suspended_state, "unblock")
+                    app.suspended_state = {}
+                    app.save()
+                blocklist.delete()
             return HttpResponse(status=204)
         except ValueError as e:
             logger.info(e)
@@ -151,20 +216,24 @@ def unblock(self, request,  **kwargs):
 
 class AdmissionWebhookViewSet(GenericViewSet):
 
+    admission_classes = (
+        admissions.JobsStatusHandler,
+        admissions.DeploymentsScaleHandler,
+        admissions.ServiceInstancesStatusHandler,
+        admissions.ServicebindingsStatusHandler,
+    )
     permission_classes = (AllowAny, )
 
-    def scale(self, request,  **kwargs):
-        token = kwargs['token']
+    def handle(self, request,  **kwargs):
+        key = kwargs['key']
         data = json.loads(request.body.decode("utf8"))["request"]
-        if settings.DRYCC_ADMISSION_WEBHOOK_TOKEN == token:
+        if settings.CERT_KEY == key:
             allowed = True
-            app_id = data["object"]["metadata"]["namespace"]
-            app = models.app.App.objects.filter(id=app_id).first()
-            replicas = data["object"]["spec"].get("replicas", 0)
-            container_type = data["object"]["metadata"]["name"].replace(f"{app_id}-", "", 1)
-            if app and app.structure.get(container_type) != replicas:  # sync replicas
-                app.structure[container_type] = replicas
-                super(models.app.App, app).save(update_fields=["structure", ])
+            for admission_class in self.admission_classes:
+                admission = admission_class()
+                if admission.detect(data):
+                    allowed = admission.handle(data)
+                    break
         else:
             allowed = False
         return Response({
@@ -177,19 +246,246 @@ def scale(self, request,  **kwargs):
         })
 
 
-class BaseDryccViewSet(viewsets.OwnerViewSet):
+class WorkspaceViewSet(ModelViewSet):
     """
-    A generic ViewSet for objects related to Drycc.
+    ViewSet for Workspace model.
+    """
+    lookup_field = 'name'
+    serializer_class = serializers.WorkspaceSerializer
+    permission_classes = [IsAuthenticated]
+
+    def get_queryset(self):
+        return models.workspace.Workspace.objects.filter(
+            workspacemember__user=self.request.user
+        ).distinct()
+
+    def perform_create(self, serializer):
+        workspace = serializer.save()
+        models.workspace.WorkspaceMember.objects.create(
+            user=self.request.user, workspace=workspace, role='admin'
+        )
+
+    def get_object(self):
+        """Override to get workspace by name instead of pk"""
+        return get_object_or_404(self.get_queryset(), name=self.kwargs['name'])
+
+    def partial_update(self, request, *args, **kwargs):
+        """Only admins can update workspaces"""
+        workspace = self.get_object()
+        if not workspace.has_member(request.user, role='admin'):
+            return Response(
+                {"detail": "Only workspace admins can update workspaces"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        return super().partial_update(request, *args, **kwargs)
+
+    def destroy(self, request, *args, **kwargs):
+        """Only admins can delete workspaces"""
+        workspace = self.get_object()
+        if not workspace.has_member(request.user, role='admin'):
+            return Response(
+                {"detail": "Only workspace admins can delete workspaces"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        if models.workspace.WorkspaceMember.objects.filter(workspace=workspace).count() > 1:
+            return Response(
+                {"detail": "Cannot delete workspace with more than one member"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        return super().destroy(request, *args, **kwargs)
 
-    To use it, at minimum you'll need to provide the `serializer_class` attribute and
-    the `model` attribute shortcut.
+
+class WorkspaceMemberViewSet(ModelViewSet):
     """
-    lookup_field = 'id'
-    permission_classes = [IsAuthenticated, permissions.IsAppUser]
-    renderer_classes = [renderers.JSONRenderer]
+    ViewSet for WorkspaceMember model.
+    """
+    serializer_class = serializers.WorkspaceMemberSerializer
+    permission_classes = [IsAuthenticated]
+
+    def get_queryset(self):
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        # Check if user has access to this workspace
+        if workspace.has_member(self.request.user):
+            return models.workspace.WorkspaceMember.objects.filter(workspace=workspace)
+        return models.workspace.WorkspaceMember.objects.none()
 
+    def get_object(self):
+        """Override to get member by username and workspace name"""
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        return get_object_or_404(
+            models.workspace.WorkspaceMember,
+            workspace=workspace, user__username=self.kwargs['user']
+        )
+
+    def partial_update(self, request, *args, **kwargs):
+        """Update a member. Admins can update any member (role and alerts).
+        Non-admins can only update their own alerts field."""
+        member = self.get_object()
+        is_admin = member.workspace.has_member(request.user, role='admin')
+
+        # Check if workspace has only one member
+        member_count = models.workspace.WorkspaceMember.objects.filter(
+            workspace=member.workspace
+        ).count()
+        is_only_member = member_count == 1
+
+        # Only member cannot modify role
+        if is_only_member and 'role' in request.data:
+            return Response(
+                {"detail": "Cannot modify role: workspace only has one member"},
+                status=status.HTTP_403_FORBIDDEN
+            )
 
-class AppResourceViewSet(BaseDryccViewSet):
+        # Non-admin users restrictions
+        if not is_admin:
+            # Cannot update other members
+            if request.user != member.user:
+                return Response(
+                    {"detail": "Only workspace admins can update other members"},
+                    status=status.HTTP_403_FORBIDDEN
+                )
+            # Cannot modify own role
+            if 'role' in request.data:
+                return Response(
+                    {"detail": "Cannot modify your own role"},
+                    status=status.HTTP_403_FORBIDDEN
+                )
+
+        return super().partial_update(request, *args, **kwargs)
+
+    def destroy(self, request, *args, **kwargs):
+        """Delete a member. Admins can delete any member.
+        Non-admins can only delete themselves (leave workspace)."""
+        member = self.get_object()
+        is_admin = member.workspace.has_member(request.user, role='admin')
+
+        # Check if workspace has only one member
+        member_count = models.workspace.WorkspaceMember.objects.filter(
+            workspace=member.workspace
+        ).count()
+        is_only_member = member_count == 1
+
+        # Only member cannot delete self
+        if is_only_member and request.user == member.user:
+            return Response(
+                {"detail": "Cannot delete: workspace only has one member"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+
+        # Non-admin can delete self
+        if request.user == member.user:
+            return super().destroy(request, *args, **kwargs)
+
+        # Admin can delete any member
+        if is_admin:
+            return super().destroy(request, *args, **kwargs)
+
+        # Other cases forbidden
+        return Response(
+            {"detail": "Only workspace admins can remove other members"},
+            status=status.HTTP_403_FORBIDDEN
+        )
+
+
+class WorkspaceInvitationViewSet(ModelViewSet):
+    """
+    ViewSet for WorkspaceInvitation model.
+    """
+    serializer_class = serializers.WorkspaceInvitationSerializer
+
+    def get_permissions(self):
+        """
+        Allow anyone to accept an invitation.
+        Only authenticated users can create or list invitations.
+        """
+        if self.action == 'retrieve':
+            return [AllowAny()]
+        return super().get_permissions()
+
+    def get_renderers(self):
+        if self.action == 'retrieve':
+            return [JSONRenderer(), TemplateHTMLRenderer()]
+        return super().get_renderers()
+
+    def get_queryset(self):
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        if workspace.has_member(self.request.user):
+            return models.workspace.WorkspaceInvitation.objects.filter(
+                workspace=workspace, accepted=False)
+        return models.workspace.WorkspaceInvitation.objects.none()
+
+    def get_object(self):
+        """Override to get invitation by uid and workspace name"""
+        return get_object_or_404(
+            models.workspace.WorkspaceInvitation,
+            workspace=get_object_or_404(models.workspace.Workspace, name=self.kwargs['name']),
+            token=self.kwargs['uid'],
+            accepted=False,
+        )
+
+    def retrieve(self, request, *args, **kwargs):
+        instance = self.get_object()
+        instance.accept()
+        user_exists = User.objects.filter(email=instance.email).exists()
+        data = {
+            'workspace_name': instance.workspace.name,
+            'user_exists': user_exists,
+            'register_url': settings.DRYCC_REGISTER_URL,
+        }
+        if isinstance(request.accepted_renderer, TemplateHTMLRenderer):
+            return render(request, 'workspace/workspace_invitation_accept.html', data)
+        return Response(data)
+
+    def perform_create(self, serializer):
+        workspace = get_object_or_404(models.workspace.Workspace, name=self.kwargs['name'])
+        if not workspace.has_member(self.request.user, role='admin'):
+            raise ValidationError("Only workspace admins can create invitations")
+        email = serializer.validated_data['email']
+        user = User.objects.filter(email=email).first()
+        if user and workspace.has_member(user):
+            raise ValidationError("User is already a member of the workspace")
+        invitation = models.workspace.WorkspaceInvitation.objects.filter(
+            email=email, workspace=workspace, accepted=False
+        ).first()
+        if not invitation:
+            models.workspace.WorkspaceInvitation.objects.filter(
+                email=email, workspace=workspace, accepted=True
+            ).delete()
+            invitation = serializer.save(
+                token=secrets.token_hex(64), inviter=self.request.user, workspace=workspace)
+        if settings.EMAIL_HOST:
+            invitation.send_email(self.request)
+        else:
+            invitation.accept()
+
+    def destroy(self, request, *args, **kwargs):
+        """Only admins can revoke invitations"""
+        invitation = self.get_object()
+        if not invitation.workspace.has_member(request.user, role='admin'):
+            return Response(
+                {"detail": "Only workspace admins can revoke invitations"},
+                status=status.HTTP_403_FORBIDDEN
+            )
+        return super().destroy(request, *args, **kwargs)
+
+
+class TokenViewSet(viewsets.OwnerViewSet):
+    """
+    A viewset for interacting with Token objects.
+    """
+    serializer_class = serializers.TokenSerializer
+
+    def get_queryset(self):
+        return models.base.Token.objects.filter(owner=self.request.user)
+
+    def destroy(self, *args, **kwargs):
+        key = self.get_object().key
+        response = super().destroy(self, *args, **kwargs)
+        cache.delete(key)
+        return response
+
+
+class AppFilterViewSet(viewsets.BaseAppViewSet):
     """A viewset for objects which are attached to an application."""
 
     def get_app(self):
@@ -206,85 +502,70 @@ def get_object(self, **kwargs):
 
     def create(self, request, **kwargs):
         request.data['app'] = self.get_app()
-        return super(AppResourceViewSet, self).create(request, **kwargs)
+        return super().create(request, **kwargs)
 
 
-class ReleasableViewSet(AppResourceViewSet):
+class ReleasableViewSet(AppFilterViewSet):
     """A viewset for application resources which affect the release cycle."""
+
     def get_object(self):
         """Retrieve the object based on the latest release's value"""
-        return getattr(
-            self.get_app().release_set.filter(failed=False).latest(),
-            self.model.__name__.lower()
-        )
+        version = self.request.query_params.get('version', '').lower().strip('v')
+        if re.search("^[0-9]+$", version):
+            release = get_object_or_404(
+                models.release.Release, app=self.get_app(), version=int(version))
+        else:
+            release = models.release.Release.latest(self.get_app())
+        return getattr(release, self.model.__name__.lower())
 
 
-class AppViewSet(BaseDryccViewSet):
+class AppViewSet(viewsets.BaseAppViewSet):
     """A viewset for interacting with App objects."""
     model = models.app.App
     filter_backends = [filters.SearchFilter]
     search_fields = ['^id', ]
     serializer_class = serializers.AppSerializer
 
-    def get_queryset(self, *args, **kwargs):
-        return self.model.objects.all(*args, **kwargs)
-
-    def list(self, request, *args, **kwargs):
-        """
-        HACK: Instead of filtering by the queryset, we limit the queryset to list only the apps
-        which are owned by the user as well as any apps they have been given permission to
-        interact with.
-        """
-        queryset = super(AppViewSet, self).get_queryset(**kwargs) | \
-            get_objects_for_user(self.request.user, 'api.use_app')
-        instance = self.filter_queryset(queryset)
-        page = self.paginate_queryset(instance)
-        if page is not None:
-            serializer = self.get_serializer(page, many=True)
-            return self.get_paginated_response(serializer.data)
-
-        serializer = self.get_serializer(instance, many=True)
-        return Response(serializer.data)
-
-    def scale(self, request, **kwargs):
-        scale_app.delay(self.get_object(), request.user, request.data)
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-    def logs(self, request, **kwargs):
-        app = self.get_object()
-        try:
-            logs = app.logs(request.query_params.get('log_lines', str(settings.LOG_LINES)))
-            return HttpResponse(logs, status=status.HTTP_200_OK, content_type='text/plain')
-        except NotFound:
-            return HttpResponse(status=status.HTTP_204_NO_CONTENT)
-        except ServiceUnavailable:
-            # TODO make 503
-            return HttpResponse("Error accessing logs for {}".format(app.id),
-                                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                                content_type='text/plain')
-
     def run(self, request, **kwargs):
         app = self.get_object()
-        if not request.data.get('command'):
-            raise DryccException("command is a required field")
+        ptype = request.data.get('ptype', 'run')
+        command = request.data.get('command', '').split()
+        timeout = int(request.data.get('timeout', 3600))
+        expires = int(request.data.get('expires', 3600))
+        if expires == 0 or expires > settings.KUBERNETES_JOB_MAX_TTL_SECONDS_AFTER_FINISHED:
+            expires = settings.KUBERNETES_JOB_MAX_TTL_SECONDS_AFTER_FINISHED
+        if not command:
+            raise DryccException('command is a required field, or it can be defined in Procfile')
+        release = models.release.Release.latest(app)
+        if release is None or release.build is None:
+            raise DryccException('no build available, please deploy a release')
         volumes = request.data.get('volumes', None)
         if volumes:
             volumes = serializers.VolumeSerializer().validate_path(volumes)
-        rc, output = app.run(self.request.user, request.data['command'], volumes)
-        return Response({'exit_code': rc, 'output': str(output)})
+        app.run(self.request.user, release.get_deploy_image(ptype),
+                command=release.get_deploy_command(ptype), args=command, volumes=volumes,
+                timeout=timeout, expires=expires)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+    def perform_create(self, serializer):
+        workspace = serializer.validated_data['workspace']
+        self.check_object_permissions(self.request, workspace)
+        serializer.save()
 
-    def update(self, request, **kwargs):
+    def transfer(self, request, **kwargs):
         app = self.get_object()
-        old_owner = app.owner
-
-        if request.data.get('owner'):
-            if self.request.user != app.owner and not self.request.user.is_superuser:
-                raise PermissionDenied()
-            new_owner = get_object_or_404(User, username=request.data['owner'])
-            # ensure all downstream objects that are owned by this user and are part of this app
-            # is also updated
-            downstream_model_owner.delay(app, old_owner, new_owner)
-        return Response(status=status.HTTP_200_OK)
+        if not app.workspace.has_member(request.user, role='admin'):
+            raise DryccException("you must be an admin of the current workspace")
+        workspace = request.data.get('workspace', '')
+        if not workspace:
+            raise DryccException("workspace is required")
+        app.workspace = get_object_or_404(models.workspace.Workspace, name=workspace)
+        app.save()
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
+    @transaction.atomic
+    def destroy(self, request, *args, **kwargs):
+        return super().destroy(request, *args, **kwargs)
 
 
 class BuildViewSet(ReleasableViewSet):
@@ -292,9 +573,49 @@ class BuildViewSet(ReleasableViewSet):
     model = models.build.Build
     serializer_class = serializers.BuildSerializer
 
-    def post_save(self, build):
-        self.release = build.create(self.request.user)
-        super(BuildViewSet, self).post_save(build)
+    def perform_create(self, serializer):
+        build = serializer.save()
+        for ptype in build.ptypes:
+            image = build.get_image(ptype)
+            if is_loopback(image):
+                raise DryccException("image must not use the loopback address")
+        build.create_release(self.request.user)
+
+
+class LimitSpecViewSet(ModelViewSet):
+    """A viewset for interacting with Limit objects."""
+    model = models.limit.LimitSpec
+    serializer_class = serializers.LimitSpecSerializer
+
+    def get_queryset(self, **kwargs):
+        q = Q(disabled=False)
+        keywords = self.request.query_params.get('keywords', '').strip()
+        if keywords:
+            q &= Q(
+                keywords__contains=[keyword.lower() for keyword in re.split(r"\W+", keywords)])
+        return self.model.objects.filter(q)
+
+
+class LimitPlanViewSet(ModelViewSet):
+    """A viewset for interacting with Limit objects."""
+    model = models.limit.LimitPlan
+    serializer_class = serializers.LimitPlanSerializer
+
+    def get_object(self):
+        return get_object_or_404(self.model, id=self.kwargs["id"])
+
+    def get_queryset(self, **kwargs):
+        q = Q(disabled=False)
+        spec_id = self.request.query_params.get('spec-id', '')
+        if spec_id:
+            q &= Q(spec_id=spec_id)
+        cpu_match = re.search("^[0-9]+", self.request.query_params.get('cpu', ''))
+        if cpu_match:
+            q &= Q(cpu=cpu_match.group())
+        memory_match = re.search("^[0-9]+", self.request.query_params.get('memory', ''))
+        if memory_match:
+            q &= Q(memory=memory_match.group())
+        return self.model.objects.filter(q)
 
 
 class ConfigViewSet(ReleasableViewSet):
@@ -302,34 +623,62 @@ class ConfigViewSet(ReleasableViewSet):
     model = models.config.Config
     serializer_class = serializers.ConfigSerializer
 
+    def create(self, request, **kwargs):
+        if self.request.query_params.get('merge', 'true').lower() == 'true':
+            return super().create(request, **kwargs)
+        values = self.get_serializer().validate_values(request.data.get('values'))
+        config = self.model(app=self.get_app(), values=values)
+        old_config = config.previous()
+        if old_config and old_config.values:
+            replace_ptypes = {v['ptype'] for v in config.values if 'ptype' in v}
+            replace_groups = {v['group'] for v in config.values if 'group' in v}
+            config.merge_field("values", old_config, replace_ptypes, replace_groups)
+        config.save(ignore_update_fields=["values"])
+        self.post_save(config)
+        data = self.get_serializer(config).data
+        headers = self.get_success_headers(data)
+        return Response(data, status=status.HTTP_201_CREATED, headers=headers)
+
+    def perform_create(self, serializer):
+        config = serializer.save()
+        self.post_save(config)
+
+    def destroy(self, request, **kwargs):
+        values_refs = self.get_serializer().validate_values_refs(
+            request.data.get('values_refs', {}))
+        if not values_refs or not values_refs.values():
+            raise DryccException("ptype or groups is required")
+
+        config = self.model(app=self.get_app(), values_refs={})
+        previous = config.previous()
+        old_values_refs = previous.values_refs.copy() if previous else {}
+        for ptype, old_groups in old_values_refs.items():
+            groups_to_delete = values_refs.get(ptype, [])
+            for group in old_groups:
+                if group not in groups_to_delete:
+                    if ptype not in config.values_refs:
+                        config.values_refs[ptype] = [group]
+                    elif group not in config.values_refs[ptype]:
+                        config.values_refs[ptype].append(group)
+        config.save(ignore_update_fields=["values_refs"])
+        self.post_save(config)
+        return Response(status=status.HTTP_200_OK)
+
     def post_save(self, config):
-        release = config.app.release_set.filter(failed=False).latest()
-        latest_version = config.app.release_set.latest().version
+        latest_release = models.release.Release.latest(self.get_app())
         try:
-            self.release = release.new(
-                self.request.user, config=config, build=release.build, canary=release.canary)
-            # It's possible to set config values before a build
-            if self.release.build is not None:
-                config.app.deploy(self.release)
-        except Exception as e:
-            if (not hasattr(self, 'release') and
-                    config.app.release_set.latest().version == latest_version+1):
-                self.release = config.app.release_set.latest()
-            if hasattr(self, 'release'):
-                self.release.failed = True
-                self.release.summary = "{} deployed a config that failed".format(
-                    self.request.user)
-                # Get the exception that has occured
-                self.release.exception = "error: {}".format(str(e))
-                self.release.save()
-            else:
-                config.delete()
+            build = latest_release.build.merge(config) if latest_release.build else None
+            release = latest_release.new(self.request.user, config=config, build=build)
+            if release.build and config.app.appsettings_set.latest().autodeploy:
+                release.deploy(release.ptypes, False)
+        except BaseException as e:
+            config.delete()
             if isinstance(e, AlreadyExists):
                 raise
             raise DryccException(str(e)) from e
 
 
-class PodViewSet(AppResourceViewSet):
+class PodViewSet(AppFilterViewSet):
     model = models.app.App
     serializer_class = serializers.PodSerializer
 
@@ -340,53 +689,96 @@ def list(self, *args, **kwargs):
         pagination = {'results': data, 'count': len(data)}
         return Response(pagination, status=status.HTTP_200_OK)
 
-    def restart(self, *args, **kwargs):
-        restart_app.delay(self.get_app(), **kwargs)
-        return Response(status=status.HTTP_204_NO_CONTENT)
+    def describe(self, *args, **kwargs):
+        pod_name = kwargs["name"]
+        data = self.get_app().describe_pod(pod_name)
+        if len(data) == 0:
+            raise DryccException("this process not found")
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
+    def destroy(self, request, **kwargs):
+        pod_names = request.data.get("pod_ids")
+        pod_names = pod_names.split(",")
+        for pod_name in set(pod_names):
+            delete_pod.delay(self.get_app(), **{"pod_name": pod_name})
+        return Response(status=status.HTTP_200_OK)
 
 
-class AppSettingsViewSet(AppResourceViewSet):
-    model = models.appsettings.AppSettings
-    serializer_class = serializers.AppSettingsSerializer
+class PtypeViewSet(AppFilterViewSet):
+    model = models.app.App
+    serializer_class = serializers.PtypeSerializer
 
-    def remove(self, request, *args, **kwargs):
+    def list(self, *args, **kwargs):
+        deploys = self.get_app().list_deployments(*args, **kwargs)
+        data = self.get_serializer(deploys, many=True).data
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
+    def describe(self, *args, **kwargs):
+        deployment_name = kwargs["name"]
+        data = self.get_app().describe_deployment(deployment_name)
+        if len(data) == 0:
+            raise DryccException("this procfile type not found")
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
+    def restart(self, request, *args, **kwargs):
         app = self.get_app()
-        app_settings = self.get_object()
-        new_canaries = []
-        remove_canaries = request.data["canaries"]
-        for procfile_tyle in app_settings.canaries:
-            if procfile_tyle not in remove_canaries:
-                new_canaries.append(procfile_tyle)
-        new_app_settings = self.model(app=app, owner=app.owner, canaries=new_canaries)
-        new_app_settings.summary = f"{app.owner} remove canaries {','.join(remove_canaries)}"
-        new_app_settings.save(ignore_update_field=["canaries"])
+        ptypes = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        ptypes = app.check_ptypes(ptypes)
+        for ptype in set(ptypes):
+            restart_app.delay(app, **{"type": ptype})
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    def clean(self, request, *args, **kwargs):
+        app = self.get_app()
+        ptypes = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        if not ptypes:
+            raise DryccException("ptypes is a required field")
+        latest_ptypes = [k for k, v in app.structure.items() if v != 0]
+        not_allow = [ptype for ptype in ptypes if ptype in latest_ptypes]
+        if not_allow:
+            raise DryccException(f'ptype {",".join(not_allow)} should not garbage.')
+        app.clean(ptypes=ptypes)
+        return Response(status=status.HTTP_204_NO_CONTENT)
 
-class CanaryViewSet(AppResourceViewSet):
-    model = models.appsettings.AppSettings
-    serializer_class = serializers.AppSettingsSerializer
-
-    def _clean_canaries(self):
+    def scale(self, request, **kwargs):
         app = self.get_app()
-        app_settings = self.model(app=app, owner=app.owner, canaries=[])
-        app_settings.summary = "canary release"
-        app_settings.save(ignore_update_field=["canaries"])
+        scale_app.delay(app, request.user, request.data)
+        return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def release(self, request, *args, **kwargs):
-        self._clean_canaries()
-        release = self.get_app().release_set.filter(failed=False).latest()
-        data = {'version': release.rollback(request.user, release.version).version}
-        return Response(data, status=status.HTTP_201_CREATED)
 
-    def rollback(self, request, *args, **kwargs):
-        self._clean_canaries()
-        release = self.get_app().release_set.filter(failed=False, canary=False).latest()
-        data = {'version': release.rollback(request.user, release.version).version}
-        return Response(data, status=status.HTTP_201_CREATED)
+class EventViewSet(AppFilterViewSet):
+    model = models.app.App
+    serializer_class = serializers.EventSerializer
 
+    def list(self, request, **kwargs):
+        ptype = request.query_params.get("ptype", None)
+        pod_name = request.query_params.get("pod_name", None)
+        if not any([ptype, pod_name]):
+            data = []
+        else:
+            ref_kind, ref_name = ("Pod", pod_name) if pod_name else \
+                ("Deployment", ptype)
+            events = self.get_app().list_events(ref_kind, ref_name)
+            data = self.get_serializer(events, many=True).data
+        # fake out pagination for now
+        pagination = {'results': data, 'count': len(data)}
+        return Response(pagination, status=status.HTTP_200_OK)
+
+
+class AppSettingsViewSet(AppFilterViewSet):
+    model = models.appsettings.AppSettings
+    serializer_class = serializers.AppSettingsSerializer
 
-class DomainViewSet(AppResourceViewSet):
+
+class DomainViewSet(AppFilterViewSet):
     """A viewset for interacting with Domain objects."""
     model = models.domain.Domain
     serializer_class = serializers.DomainSerializer
@@ -411,7 +803,7 @@ def destroy(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class ServiceViewSet(AppResourceViewSet):
+class ServiceViewSet(AppFilterViewSet):
     """A viewset for interacting with Service objects."""
     model = models.service.Service
     serializer_class = serializers.ServiceSerializer
@@ -425,42 +817,37 @@ def create_or_update(self, request, **kwargs):
         app = self.get_app()
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data.get('protocol'))
-        procfile_type = self.get_serializer().validate_procfile_type(request.data.get(
-            'procfile_type'))
+        ptype = self.get_serializer().validate_ptype(request.data.get(
+            'ptype'))
         target_port = self.get_serializer().validate_target_port(request.data.get('target_port'))
-        service = app.service_set.filter(procfile_type=procfile_type).first()
+        service = app.service_set.filter(ptype=ptype).first()
         if service:
             for item in service.ports:
                 if item["port"] == port:
                     return Response(status=status.HTTP_400_BAD_REQUEST, data={"detail": "port is occupied"})  # noqa
             http_status = status.HTTP_204_NO_CONTENT
         else:
-            service = self.model(owner=app.owner, app=app, procfile_type=procfile_type)
+            service = self.model(app=app, ptype=ptype)
             http_status = status.HTTP_201_CREATED
         service.add_port(port, protocol, target_port)
-        service.canary = procfile_type in app.appsettings_set.latest().canaries
         service.save()
         return Response(status=http_status)
 
-    def delete(self, request, **kwargs):
+    def destroy(self, request, **kwargs):
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data.get('protocol'))
-        procfile_type = self.get_serializer().validate_procfile_type(
-            request.data.get('procfile_type'))
-        service = get_object_or_404(self.get_queryset(**kwargs), procfile_type=procfile_type)
-        ports = []
-        for item in service.ports:
-            if item["port"] != port or item["protocol"] != protocol:
-                ports.append(item)
-        if len(ports) == 0:
+        ptype = self.get_serializer().validate_ptype(
+            request.data.get('ptype'))
+        service = get_object_or_404(self.get_queryset(**kwargs), ptype=ptype)
+        removed = service.remove_port(port, protocol)
+        if len(service.ports) == 0:
             service.delete()
-        elif len(ports) < len(service.ports):
-            service.ports = ports
+        elif removed:
             service.save()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class CertificateViewSet(BaseDryccViewSet):
+class CertificateViewSet(AppFilterViewSet):
     """A viewset for interacting with Certificate objects."""
     model = models.certificate.Certificate
     serializer_class = serializers.CertificateSerializer
@@ -488,18 +875,17 @@ def detach(self, request, *args, **kwargs):
             self.get_object().detach(*args, **kwargs)
         except Http404:
             raise
-
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class KeyViewSet(BaseDryccViewSet):
+class KeyViewSet(viewsets.OwnerViewSet):
     """A viewset for interacting with Key objects."""
+    lookup_field = 'id'
     model = models.key.Key
-    permission_classes = [IsAuthenticated, permissions.IsOwner]
     serializer_class = serializers.KeySerializer
 
 
-class ReleaseViewSet(AppResourceViewSet):
+class ReleaseViewSet(AppFilterViewSet):
     """A viewset for interacting with Release objects."""
     model = models.release.Release
     serializer_class = serializers.ReleaseSerializer
@@ -509,24 +895,58 @@ def get_object(self, **kwargs):
         qs = self.get_queryset(**kwargs)
         return get_object_or_404(qs, version=self.kwargs['version'])
 
+    def get_queryset(self, **kwargs):
+        ptypes = self.request.query_params.get('ptypes', '').strip()
+        queryset = super().get_queryset(**kwargs)
+        if ptypes:
+            queryset = queryset.filter(Q(
+                deployed_ptypes__contains=[
+                    ptype.lower() for ptype in re.split(r"\W+", ptypes)]))
+        return queryset
+
+    def deploy(self, request, **kwargs):
+        """Deploy the latest release"""
+        latest_release = self.get_app().release_set.latest()
+
+        force_deploy = request.data.get("force", False)
+        ptypes = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        if not ptypes:
+            ptypes = latest_release.ptypes
+        else:
+            invalid_ptypes = ptypes.difference(
+                latest_release.ptypes + [d["name"] for d in self.get_app().list_deployments()])
+            if len(invalid_ptypes) != 0:
+                raise DryccException(f"process type {','.join(invalid_ptypes)} is not exists")
+        latest_release.deploy(ptypes, force_deploy)
+        return Response(status=status.HTTP_204_NO_CONTENT)
+
     def rollback(self, request, **kwargs):
         """
         Create a new release as a copy of the state of the compiled slug and config vars of a
         previous release.
         """
-        release = self.get_app().release_set.filter(failed=False).latest()
-        new_release = release.rollback(request.user, request.data.get('version', None))
+        latest_release = models.release.Release.latest(self.get_app())
+        ptypes = set(
+            [ptype for ptype in request.data.get("ptypes", "").split(",") if ptype])
+        ptypes = latest_release.app.check_ptypes(ptypes)
+        new_release = latest_release.rollback(
+            request.user, ptypes, request.data.get('version', None))
         response = {'version': new_release.version}
         return Response(response, status=status.HTTP_201_CREATED)
 
 
-class TLSViewSet(AppResourceViewSet):
+class TLSViewSet(AppFilterViewSet):
     model = models.tls.TLS
     serializer_class = serializers.TLSSerializer
 
+    def events(self, request, **kwargs):
+        results = self.get_object().events()
+        return Response({'results': results, 'count': len(results)})
+
 
-class BaseHookViewSet(BaseDryccViewSet):
-    permission_classes = [permissions.HasBuilderAuth]
+class BaseHookViewSet(viewsets.BaseAppViewSet):
+    permission_classes = [permissions.IsServiceToken]
 
 
 class KeyHookViewSet(BaseHookViewSet):
@@ -537,32 +957,22 @@ class KeyHookViewSet(BaseHookViewSet):
     def public_key(self, request, *args, **kwargs):
         fingerprint = kwargs['fingerprint'].strip()
         key = get_object_or_404(models.key.Key, fingerprint=fingerprint)
-
-        queryset = models.app.App.objects.all() | \
-            get_objects_for_user(self.request.user, 'api.use_app')
-        items = self.filter_queryset(queryset)
-
-        apps = []
-        for item in items:
-            apps.append(item.id)
-
+        queryset = models.app.App.objects.filter(
+            workspace__workspacemember__user=self.request.user
+        ).distinct()
         data = {
             'username': key.owner.username,
-            'apps': apps
+            'apps': [item.id for item in self.filter_queryset(queryset)],
         }
 
         return Response(data, status=status.HTTP_200_OK)
 
     def app(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
-
-        perm_name = "api.use_app"
-        usernames = [u.id for u in get_users_with_perms(app)
-                     if u.has_perm(perm_name, app)]
-
+        usernames = app.workspace.workspacemember_set.values_list('user__username', flat=True)
         data = {}
         result = models.key.Key.objects \
-                       .filter(owner__in=usernames) \
+                       .filter(owner__username__in=usernames) \
                        .values('owner__username', 'public', 'fingerprint') \
                        .order_by('created')
         for info in result:
@@ -581,9 +991,8 @@ def users(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=kwargs['id'])
         request.user = get_object_or_404(User, username=kwargs['username'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request, app)
-        if not has_permission:
-            raise PermissionDenied(message)
+        if not permissions.IsAppUser().has_object_permission(request, self, app):
+            return Response(status=status.HTTP_403_FORBIDDEN)
 
         data = {request.user.username: []}
         keys = models.key.Key.objects \
@@ -591,7 +1000,7 @@ def users(self, request, *args, **kwargs):
                      .values('public', 'fingerprint') \
                      .order_by('created')
         if not keys:
-            raise NotFound("No Keys match the given query.")
+            return Response("No Keys match the given query.", status=status.HTTP_404_NOT_FOUND)
 
         for info in keys:
             data[request.user.username].append({
@@ -611,22 +1020,22 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         self.user = request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request, app)
-        if not has_permission:
-            raise PermissionDenied(message)
+        if not permissions.IsAppUser().has_object_permission(request, self, app):
+            return Response(status=status.HTTP_403_FORBIDDEN)
         request.data['app'] = app
         request.data['owner'] = self.user
-        super(BuildHookViewSet, self).create(request, *args, **kwargs)
+        super().create(request, *args, **kwargs)
         # return the application databag
         response = {
             'release': {
-                'version': app.release_set.filter(failed=False).latest().version
+                'version': models.release.Release.latest(app).version
             }
         }
         return Response(response, status=status.HTTP_200_OK)
 
-    def post_save(self, build):
-        build.create(self.user)
+    def perform_create(self, serializer):
+        build = serializer.save()
+        build.create_release(self.user)
 
 
 class ConfigHookViewSet(BaseHookViewSet):
@@ -638,59 +1047,14 @@ def create(self, request, *args, **kwargs):
         app = get_object_or_404(models.app.App, id=request.data['receive_repo'])
         request.user = get_object_or_404(User, username=request.data['receive_user'])
         # check the user is authorized for this app
-        has_permission, message = permissions.has_app_permission(request, app)
-        if not has_permission:
-            raise PermissionDenied(message)
-        config = app.release_set.filter(failed=False).latest().config
+        if not permissions.IsAppUser().has_object_permission(request, self, app):
+            return Response(status=status.HTTP_403_FORBIDDEN)
+        config = models.release.Release.latest(app).config
         serializer = self.get_serializer(config)
         return Response(serializer.data, status=status.HTTP_200_OK)
 
 
-class AppPermsViewSet(BaseDryccViewSet):
-    """RESTful views for sharing apps with collaborators."""
-
-    model = models.app.App  # models class
-    perm = 'use_app'    # short name for permission
-
-    def get_queryset(self):
-        return self.model.objects.all()
-
-    def list(self, request, **kwargs):
-        app = self.get_object()
-        perm_name = "api.{}".format(self.perm)
-        usernames = [u.username for u in get_users_with_perms(app)
-                     if u.has_perm(perm_name, app)]
-        return Response({'users': usernames})
-
-    def create(self, request, **kwargs):
-        app = self.get_object()
-        if not permissions.IsOwnerOrAdmin.has_object_permission(permissions.IsOwnerOrAdmin(),
-                                                                request, self, app):
-            raise PermissionDenied()
-
-        user = get_object_or_404(User, username=request.data['username'])
-        assign_perm(self.perm, user, app)
-        app.log("User {} was granted access to {}".format(user, app))
-        return Response(status=status.HTTP_201_CREATED)
-
-    def destroy(self, request, **kwargs):
-        app = get_object_or_404(models.app.App, id=self.kwargs['id'])
-        user = get_object_or_404(User, username=kwargs['username'])
-
-        perm_name = "api.{}".format(self.perm)
-        if not user.has_perm(perm_name, app):
-            raise PermissionDenied()
-
-        if (user != request.user and
-            not permissions.IsOwnerOrAdmin.has_object_permission(permissions.IsOwnerOrAdmin(),
-                                                                 request, self, app)):
-            raise PermissionDenied()
-        remove_perm(self.perm, user, app)
-        app.log("User {} was revoked access to {}".format(user, app))
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-class AppVolumesViewSet(ReleasableViewSet):
+class AppVolumesViewSet(AppFilterViewSet):
     """RESTful views for volumes apps with collaborators."""
     model = models.volume.Volume
     serializer_class = serializers.VolumeSerializer
@@ -701,72 +1065,53 @@ def get_object(self):
                                  name=self.kwargs['name'])
 
     def expand(self, request, **kwargs):
-        volume = self.get_object()
-        volume.expand(request.data['size'])
-        serializer = self.get_serializer(volume, many=False)
-        return Response(serializer.data)
+        size, volume = request.data['size'], self.get_object()
+        if volume.type == "csi":
+            if utils.unit_to_bytes(request.data['size']) < utils.unit_to_bytes(volume.size):
+                raise DryccException('Shrink volume is not supported.')
+            volume.size = size
+            volume.save()
+            serializer = self.get_serializer(volume, many=False)
+            return Response(serializer.data)
+        raise DryccException(f'{volume.type} volume is not support expand.')
 
     def destroy(self, request, **kwargs):
         volume = self.get_object()
-        if volume.path != {}:
+        app = self.get_app()
+        is_subset = set(volume.path.keys()).issubset(set(app.ptypes))
+        if volume.path != {} and is_subset:
             raise DryccException("this volume is mounting")
         volume.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
     def path(self, request, *args, **kwargs):
-        new_path = request.data.get('path')
-        if new_path is None:
+        path = request.data.get('path')
+        if path is None:
             raise DryccException("path is a required field")
         else:
-            new_path = serializers.VolumeSerializer().validate_path(new_path)
+            path = serializers.VolumeSerializer().validate_path(path)
         volume = self.get_object()
-        container_types = [_ for _ in new_path.keys()
-                           if _ not in volume.app.types]
-        if container_types:
-            raise DryccException("process type {} is not included in profile".
-                                 format(','.join(container_types)))
-
-        if set(new_path.items()).issubset(set(volume.path.items())):
+        ptypes = [_ for _ in path.keys() if _ not in volume.app.ptypes]
+        if ptypes:
+            raise DryccException("process type {} is not included in procfile".
+                                 format(','.join(ptypes)))
+        if set(path.items()).issubset(set(volume.path.items())):
             raise DryccException("mount path not changed")
+        volume.check_path(path)
 
-        other_volumes = self.get_app().volume_set.exclude(name=volume.name)
-        type_paths = {}  # {'type1':[path1,path2], tyep2:[path3,path4]}
-        for _ in other_volumes:
-            for k, v in _.path.items():
-                if k not in type_paths:
-                    type_paths[k] = [v]
-                else:
-                    type_paths[k].append(v)
-        repeat_path = [v for k, v in new_path.items() if v in type_paths.get(k, [])]  # noqa
-        if repeat_path:
-            raise DryccException("path {} is used by another volume".
-                                 format(','.join(repeat_path)))
-        path = volume.path
-        # merge mount path
-        # remove path keys if a null value is provided
-        for key, value in new_path.items():
-            if value is None:
-                # error if unsetting non-existing key
-                if key not in path:
-                    raise UnprocessableEntity(
-                        '{} does not exist under {}'.format(key, "volume"))  # noqa
-                path.pop(key)
-            else:
-                path[key] = value
         app = self.get_app()
-        volume.path = path  # volume save by task success
-        mount_app.delay(app, self.request.user, volume)
+        mount_app.delay(app, self.request.user, volume, path)
         serializer = self.get_serializer(volume, many=False)
         return Response(serializer.data)
 
 
-class AppResourcesViewSet(AppResourceViewSet):
+class AppResourcesViewSet(AppFilterViewSet):
     """RESTful views for resources apps with collaborators."""
     model = models.resource.Resource
     serializer_class = serializers.ResourceSerializer
 
     def services(self, request, *args, **kwargs):
-        results = self.model.services
+        results = self.model.services()
         # fake out pagination for now
         pagination = {'results': results, 'count': len(results)}
         return Response(data=cache.get_or_set(
@@ -783,7 +1128,7 @@ def plans(self, request, *args, **kwargs):
         ))
 
 
-class AppSingleResourceViewSet(AppResourceViewSet):
+class AppSingleResourceViewSet(AppFilterViewSet):
     """RESTful views for resource apps with collaborators."""
     model = models.resource.Resource
     serializer_class = serializers.ResourceSerializer
@@ -796,7 +1141,7 @@ def get_object(self):
     def retrieve(self, request, *args, **kwargs):
         resource = self.get_object()
         resource.retrieve(request)
-        response =  super(AppSingleResourceViewSet, self).retrieve(request, *args, **kwargs)  # noqa
+        response =  super().retrieve(request, *args, **kwargs)  # noqa
         response.data["message"] = resource.message
         return response
 
@@ -815,7 +1160,7 @@ def update(self, request, *args, **kwargs):
         return Response(resource.data)
 
 
-class AppResourceBindingViewSet(AppResourceViewSet):
+class AppResourceBindingViewSet(AppFilterViewSet):
     model = models.resource.Resource
     serializer_class = serializers.ResourceSerializer
 
@@ -839,10 +1184,10 @@ def binding(self, request, *args, **kwargs):
             logger.info("resoruce unbind response data: {}".format(serializer))
             return Response(serializer.data)
         else:
-            return Http404("unknown action")
+            return Response("unknown action", status=status.HTTP_404_NOT_FOUND)
 
 
-class GatewayViewSet(AppResourceViewSet):
+class GatewayViewSet(AppFilterViewSet):
     """A viewset for interacting with Gateway objects."""
     model = models.gateway.Gateway
     filter_backends = [filters.SearchFilter]
@@ -856,14 +1201,14 @@ def create_or_update(self, request, *args, **kwargs):
         protocol = self.get_serializer().validate_protocol(request.data['protocol'])
         gateway = app.gateway_set.filter(name=name).first()
         if not gateway:
-            gateway = self.model(app=app, owner=app.owner, name=name)
+            gateway = self.model(app=app, name=name)
         added, msg = gateway.add(port, protocol)
         if not added:
             return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
         gateway.save()
         return Response(status=status.HTTP_201_CREATED)
 
-    def delete(self, request, **kwargs):
+    def destroy(self, request, **kwargs):
         app = self.get_app()
         port = self.get_serializer().validate_port(request.data.get('port'))
         protocol = self.get_serializer().validate_protocol(request.data['protocol'])
@@ -878,7 +1223,7 @@ def delete(self, request, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class RouteViewSet(AppResourceViewSet):
+class RouteViewSet(AppFilterViewSet):
     """A viewset for interacting with Route objects."""
     model = models.gateway.Route
     filter_backends = [filters.SearchFilter]
@@ -896,46 +1241,17 @@ def set(self, request, *args, **kwargs):
         rules = request.data
         if isinstance(rules, str):
             rules = json.loads(rules)
-        rules = self.get_serializer().validate_rules(rules)
-        route.rules = rules
-        ok, msg = route.check_rules()
+        rules = self.get_serializer(route, many=False).validate_rules(rules)
+        ok, msg = route.check_rules(rules)
         if not ok:
             return Response(status=status.HTTP_400_BAD_REQUEST, data=msg)
+        route.rules = rules
         route.save()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def create(self, request, *args, **kwargs):
-        app = self.get_app()
-        port = self.get_serializer().validate_port(request.data.get('port'))
-        app_settings = app.appsettings_set.latest()
-        procfile_type = self.get_serializer().validate_procfile_type(
-            request.data.get('procfile_type'))
-        kind = self.get_serializer().validate_kind(request.data.get('kind'))
-        name = request.data['name']
-        route = app.route_set.filter(name=name).first()
-        if route:
-            return Response(status=status.HTTP_400_BAD_REQUEST, data={
-                "detail": f"this route {name} already exists"})
-        route = self.model(
-            app=app,
-            owner=app.owner,
-            kind=kind,
-            name=name,
-            port=port,
-            routable=app_settings.routable,
-            procfile_type=procfile_type,
-        )
-        route.rules = route.default_rules
-        if not route.current_rules[0]["backendRefs"]:
-            return Response(status=status.HTTP_400_BAD_REQUEST, data={
-                "detail": "this route does not match services. please add service first."
-            })
-        route.save()
-        return Response(status=status.HTTP_201_CREATED)
-
     def attach(self, request, *args, **kwargs):
         app = self.get_app()
-        port = self.get_serializer().validate_port(request.data.get('port'))
+        port = serializers.validate_port(request.data.get('port'))
         gateway_name = request.data['gateway']
         route = get_object_or_404(self.model, app=app, name=kwargs['name'])
         attached, msg = route.attach(gateway_name, port)
@@ -946,7 +1262,7 @@ def attach(self, request, *args, **kwargs):
 
     def detach(self, request, *args, **kwargs):
         app = self.get_app()
-        port = self.get_serializer().validate_port(request.data.get('port'))
+        port = serializers.validate_port(request.data.get('port'))
         gateway_name = request.data['gateway']
         route = get_object_or_404(self.model, app=app, name=kwargs['name'])
         detached, msg = route.detach(gateway_name, port)
@@ -955,64 +1271,14 @@ def detach(self, request, *args, **kwargs):
         route.save()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
-    def delete(self, request, *args, **kwargs):
+    def destroy(self, request, *args, **kwargs):
         app = self.get_app()
         route = get_object_or_404(self.model, app=app, name=kwargs['name'])
         route.delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class AdminPermsViewSet(BaseDryccViewSet):
-    """RESTful views for sharing admin permissions with other users."""
-
-    model = User
-    serializer_class = serializers.AdminUserSerializer
-    permission_classes = [permissions.IsAdmin]
-
-    def get_queryset(self, **kwargs):
-        self.check_object_permissions(self.request, self.request.user)
-        return self.model.objects.filter(is_active=True, is_superuser=True)
-
-    def create(self, request, **kwargs):
-        user = get_object_or_404(self.model, username=request.data['username'])
-        user.is_superuser = user.is_staff = True
-        user.save(update_fields=['is_superuser', 'is_staff'])
-        return Response(status=status.HTTP_201_CREATED)
-
-    def destroy(self, request, **kwargs):
-        user = get_object_or_404(self.model, username=kwargs['username'])
-        user.is_superuser = user.is_staff = False
-        user.save(update_fields=['is_superuser', 'is_staff'])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-class UserView(BaseDryccViewSet):
-    """A Viewset for interacting with User objects."""
-    model = User
-    serializer_class = serializers.UserSerializer
-    permission_classes = [permissions.IsAdmin]
-
-    def get_queryset(self):
-        return self.model.objects.exclude(username='AnonymousUser')
-
-    def enable(self, request, **kwargs):
-        if request.user.username == kwargs['username']:
-            return Response(status=status.HTTP_423_LOCKED)
-        user = get_object_or_404(self.model, username=kwargs['username'])
-        user.is_active = True
-        user.save(update_fields=['is_active', ])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-    def disable(self, request, **kwargs):
-        if request.user.username == kwargs['username']:
-            return Response(status=status.HTTP_423_LOCKED)
-        user = get_object_or_404(self.model, username=kwargs['username'])
-        user.is_active = False
-        user.save(update_fields=['is_active', ])
-        return Response(status=status.HTTP_204_NO_CONTENT)
-
-
-class MetricView(BaseDryccViewSet):
+class MetricView(viewsets.BaseAppViewSet):
     """Getting monitoring indicators from monitor database"""
 
     def _get_app(self):
@@ -1020,72 +1286,243 @@ def _get_app(self):
         self.check_object_permissions(self.request, app)
         return app
 
-    def _get_cpus(self, app_id, container_type, start, stop, every):
-        avg_list, max_list = [], []
-        for _, _, _, timestamp, max, avg in monitor.query_cpu_usage(
-                app_id, container_type, start, stop, every):
-            max_list.append((timestamp, max))
-            avg_list.append((timestamp, avg))
-        return {
-            "max": max_list,
-            "avg": avg_list,
-        }
+    @method_decorator(cache_page(settings.DRYCC_METRICS_EXPIRY))
+    @method_decorator(vary_on_headers("Authorization"))
+    def metric(self, request, **kwargs):
+        warnings.warn(
+            'this interface will be removed in the next version.', PendingDeprecationWarning)
+        app_id = self._get_app().id
+        return StreamingHttpResponse(
+            streaming_content=monitor.last_metrics(app_id)
+        )
 
-    def _get_memory(self, app_id, container_type, start, stop, every):
-        max_list, avg_list = [], []
-        for _, _, _, timestamp, max, avg in monitor.query_memory_usage(
-                app_id, container_type, start, stop, every):
-            max_list.append((timestamp, max))
-            avg_list.append((timestamp, avg))
-        return {
-            "max": max_list,
-            "avg": avg_list,
-        }
 
-    def _get_networks(self, app_id, container_type, start, stop, every):
-        networks = []
-        for _, _, timestamp, rx_bytes, tx_bytes in monitor.query_network_usage(
-                app_id, container_type, start, stop, every):
-            networks.append((timestamp, rx_bytes, tx_bytes))
-        return networks
+class MetricsProxyView(View):
+    cache = {}
+    match_meta = staticmethod(
+        re.compile(r'^(?:# (?:HELP|TYPE) )([a-zA-Z_][a-zA-Z0-9_:.-]*)').match)
+    match_data = staticmethod(
+        re.compile(r'^([a-zA-Z_][a-zA-Z0-9_:]*)(?:\{([^}]*)\})?\s+(\S+)').match)
+    default_cache_value = (None, -1)
 
-    def status(self, request, **kwargs):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        if settings.K8S_API_VERIFY_TLS:
+            ssl_context = ssl.create_default_context(
+                cafile='/var/run/secrets/kubernetes.io/serviceaccount/ca.crt')
+        else:
+            ssl_context = ssl.create_default_context()
+        self.connector = aiohttp.TCPConnector(ssl_context=ssl_context)
+
+    async def sample(self, name, labels_str, value):
+        if not labels_str or name not in settings.DRYCC_METRICS_CONFIG:
+            return None
+        fields = set(settings.DRYCC_METRICS_CONFIG[name])
+        labels = {}
+        for pair in labels_str.strip(" {}").split(','):
+            if '=' not in pair:
+                continue
+            k, v = pair.split('=', 1)
+            if k in fields:
+                labels[k] = v.strip(' "')
+        app_id = labels.get("namespace", labels.get(settings.DRYCC_METRICS_CONFIG[name][0]))
+        if not app_id:
+            return None
+        workspace_id, timeout = self.cache.get(app_id, self.default_cache_value)
+        if workspace_id is None or time.time() > timeout:
+            if app := await models.app.App.objects.filter(id=app_id).afirst():
+                workspace_id = app.workspace_id
+            else:
+                workspace_id = None
+            self.cache[app_id] = (workspace_id, time.time() + random.randint(600, 1200))
+        if workspace_id is None:
+            return None
+        project_id = zlib.crc32(app_id.encode("utf-8"))
+        labels.update({'vm_account_id': workspace_id, 'vm_project_id': project_id})
+        return "%s{%s} %s\n" % (name, ",".join([f'{k}="{v}"' for k, v in labels.items()]), value)
+
+    async def get(self, request):
+        params = dict(request.GET)
+        if not set(["host", "port"]).issubset(params.keys()):
+            return HttpResponse(
+                "Error: Required parameter 'host' or 'port' is missing or empty", status=400)
+        host, port = params.pop('host')[0], params.pop('port')[0]
+        scheme, path = params.pop('scheme', ['http'])[0], params.pop('path', ['/metrics'])[0]
+        url = urljoin(f"{scheme}://{host}:{port}", path)
+        headers = {"Authorization": request.META.get("HTTP_AUTHORIZATION", "")}
+
+        async def stream_response():
+            async with aiohttp.ClientSession(connector=self.connector) as session:
+                async with session.get(url, params=params, headers=headers) as resp:
+                    async for line_bytes in resp.content:
+                        line = line_bytes.decode('utf-8', errors='ignore').strip(' \n')
+                        if line.startswith('#') and (match := self.match_meta(line)):
+                            if match.group(1) in settings.DRYCC_METRICS_CONFIG:
+                                yield f"{line}\n"
+                            continue
+                        match = self.match_data(line)
+                        if not match:
+                            continue
+                        name, labels_str, value = match.groups()
+                        sample = await self.sample(name, labels_str, value)
+                        if not sample:
+                            continue
+                        yield sample
+        content_type = f"text/plain; version={__version__}"
+        return StreamingHttpResponse(stream_response(), content_type=content_type)
+
+
+@method_decorator(csrf_exempt, name='dispatch')
+class BaseUserProxyView(View):
+    timeout = aiohttp.ClientTimeout(total=30, connect=10, sock_read=15)
+    permission = permissions.IsServiceToken()
+    authentication = authentication.DryccAuthentication()
+    authentication.ignore_authentication_failed = True
+
+    async def authenticate(self, request, username):
         """
-        {
-            "id": "django_t1",
-            "type": "web",
-            "count": 1,
-            "status": {
-                "cpus": {
-                    "max": [(1611023853, 50000)],
-                    "avg": [(1611023853, 50000)]
-                },
-                "memory": {
-                    "max": [(1611023853, 50000)],
-                    "avg": [(1611023853, 50000)],
-                },
-                "networks": [
-                    (1611023853, 10000, 50000)
-                ]
-            }
-        }
+        Authenticate the user based on the provided request and username.
+        Returns the user ID on success; returns None and an error message on failure.
         """
-        app_id = self._get_app().id
-        data = serializers.MetricSerializer(data=self.request.query_params)
-        if not data.is_valid():
-            return Response(data.errors, status=status.HTTP_422_UNPROCESSABLE_ENTITY)
-        start, stop, every = data.validated_data['start'], data.validated_data[
-            'stop'], data.validated_data["every"]
-        return Response({
-            "id": app_id,
-            "type": kwargs['type'],
-            "count": monitor.query_container_count(app_id, kwargs['type'], start, stop),
-            "status": {
-                "cpus": self._get_cpus(
-                    app_id, kwargs['type'], start, stop, every),
-                "memory": self._get_memory(
-                    app_id, kwargs['type'], start, stop, every),
-                "networks": self._get_networks(
-                    app_id, kwargs['type'], start, stop, every),
-            }
-        })
+        if self.permission.has_permission(request, None):
+            if username != "drycc":
+                return None, {'error': 'Access denied', "status_code": 403}
+            return 0, None
+        auth = await database_sync_to_async(self.authentication.authenticate)(request)
+        if not auth or len(auth) != 2:
+            return None, {'error': 'Unauthorized', "status_code": 401}
+        if auth[0].username != username:
+            return None, {'error': 'Access denied', "status_code": 403}
+        return auth[0].id, None
+
+
+@method_decorator(csrf_exempt, name='dispatch')
+class QuickwitProxyView(BaseUserProxyView):
+    index_url_match = re.compile(r"^indexes/?$").match
+    search_url_match = re.compile(r"^(?P<index>[a-zA-Z*][\w.*-，]{0,})/search/?$").match
+    msearch_url_match = re.compile(r"^_elastic/_msearch/?$").match
+    field_caps_url_match = re.compile(
+        r"_elastic/(?P<index>[a-zA-Z*][\w.*-，]{0,})/_field_caps/?$").match
+
+    async def proxy(self, request, username, path):
+        user_id, message = await self.authenticate(request, username)
+        if user_id is None and message is not None:
+            return JsonResponse(message, status=message["status_code"])
+        kwargs = {"request": request, "username": username}
+        if self.index_url_match(path):
+            func, kwargs["index"] = self.index, request.GET.get("index_id_patterns", "*")
+        elif match := self.search_url_match(path):
+            func, kwargs["index"] = self.query, match.group("index")
+        elif self.msearch_url_match(path):
+            func = self.msearch
+        elif match := self.field_caps_url_match(path):
+            func, kwargs["index"] = self.field_caps, match.group("index")
+        else:
+            return JsonResponse({'error': 'Not Found'}, status=404)
+        return await func(**kwargs)
+
+    async def index(self, request, username, index):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        index = await self.get_app_indexes(username, index)
+        url, params = urljoin(base_url, "/api/v1/indexes"), dict(request.GET)
+        params["index_id_patterns"] = index
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status, safe=False)
+
+    async def query(self, request, username, index):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        index = await self.get_app_indexes(username, index)
+        url, params = urljoin(base_url, f"/api/v1/{index}/search"), dict(request.GET)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    async def msearch(self, request, username):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        json_lines = request.body.decode('utf-8').strip().split('\n')
+        for i, json_line in enumerate(json_lines):
+            if i % 2 == 0:
+                request_header = json.loads(json_line)
+                request_header['index'] = ",".join(
+                    [await self.get_app_indexes(username, i) for i in request_header['index']]
+                ).split(",")
+                json_lines[i] = json.dumps(request_header)
+        url, params = urljoin(
+            base_url, "/api/v1/_elastic/_msearch"), dict(request.GET)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    url, data="\n".join(json_lines), params=params, timeout=self.timeout
+                ) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    async def field_caps(self, request, username, index):
+        base_url = settings.QUICKWIT_SEARCHER_URL
+        index = await self.get_app_indexes(username, index)
+        url, params = urljoin(
+            base_url, f"/api/v1/_elastic/{index}/_field_caps"), dict(request.GET)
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url, params=params, timeout=self.timeout) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'quickwit connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    async def get_app_indexes(self, username, index):
+        if username == "drycc":
+            return index
+        if "," in index:
+            match = re.compile("|".join([f"^{i}$" for i in index.split(",")])).match
+        else:
+            match = re.compile(index).match
+        app_indexes, log_index_prefix = [], settings.QUICKWIT_LOG_INDEX_PREFIX
+        if hasattr(self, "app_ids"):
+            app_ids = self.app_ids
+        else:
+            app_ids = [
+                app.id async for app in models.app.App.objects.filter(owner__username=username)]
+            setattr(self, "app_ids", app_ids)
+        for app_id in app_ids:
+            app_index = f"{log_index_prefix}{app_id}"
+            if match(app_index):
+                app_indexes.append(app_index)
+        return ",".join(app_indexes)
+
+    get = post = proxy
+
+
+@method_decorator(csrf_exempt, name='dispatch')
+class PrometheusProxyView(BaseUserProxyView):
+    async def proxy(self, request, username, path):
+        user_id, message = await self.authenticate(request, username)
+        if user_id is None and message is not None:
+            return JsonResponse(message, status=message["status_code"])
+        data = dict(request.GET) if request.method == "GET" else dict(request.POST)
+        data['extra_filters[]'] = '{vm_account_id="%s"}' % user_id
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    f"{settings.DRYCC_VICTORIAMETRICS_URL.rstrip("/")}/{path}",
+                    data=data,
+                    headers={"Content-Type": "application/x-www-form-urlencoded"},
+                    timeout=self.timeout
+                ) as response:
+                    data, status = await response.json(), response.status
+        except aiohttp.ClientError as e:
+            data, status = {'error': f'victoriametrics connection failed: {str(e)}'}, 502
+        return JsonResponse(data, status=status)
+
+    get = post = proxy
diff --git a/rootfs/api/viewsets.py b/rootfs/api/viewsets.py
index 4e972fdd5..0871f5ba7 100644
--- a/rootfs/api/viewsets.py
+++ b/rootfs/api/viewsets.py
@@ -1,4 +1,5 @@
-from rest_framework import viewsets
+from django.core.exceptions import ImproperlyConfigured
+from rest_framework import viewsets, renderers
 from rest_framework.permissions import IsAuthenticated
 
 from api import permissions
@@ -17,12 +18,25 @@ def get_queryset(self):
         return self.model.objects.filter(owner=self.request.user)
 
     def perform_create(self, serializer):
-        obj = serializer.save(owner=self.request.user)
-        self.post_save(obj)
-
-    def post_save(self, obj):
-        """
-        A post_save hook for performing actions after the object has been pushed to the database.
-        Leave it up to child classes to implement.
-        """
-        pass
+        serializer.save(owner=self.request.user)
+
+
+class BaseAppViewSet(viewsets.ModelViewSet):
+    """
+    A ViewSet for the Workspace model, which filters workspaces by membership and role.
+    """
+    lookup_field = 'id'
+    permission_classes = [IsAuthenticated, permissions.IsAppUser]
+    renderer_classes = [renderers.JSONRenderer]
+
+    def get_queryset(self):
+        # Prefer direct workspace relation, then support app->workspace chain.
+        if hasattr(self.model, 'workspace'):
+            return self.model.objects.filter(
+                workspace__workspacemember__user=self.request.user).distinct()
+        elif hasattr(self.model, 'app'):
+            return self.model.objects.filter(
+                app__workspace__workspacemember__user=self.request.user).distinct()
+        raise ImproperlyConfigured(
+            f"{self.__class__.__name__} requires a model with a 'workspace' or 'app' field."
+        )
diff --git a/rootfs/bin/boot b/rootfs/bin/boot
index 4abfe69a7..d1c2bf08f 100755
--- a/rootfs/bin/boot
+++ b/rootfs/bin/boot
@@ -23,10 +23,6 @@ echo ""
 echo "Health Checks:"
 python /workspace/manage.py healthchecks
 
-echo ""
-echo "Database Migrations:"
-python /workspace/manage.py cluster_migrate --noinput
-
 # spawn a gunicorn server in the background
 echo ""
 echo "Starting up Gunicorn"
diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit
index 3afb77c1d..218a94568 100755
--- a/rootfs/bin/test-unit
+++ b/rootfs/bin/test-unit
@@ -6,21 +6,13 @@
 # fail hard and fast even on pipelines
 set -eou pipefail
 
-function start_redis() {
+function start_valkey() {
     cd /tmp
-    nohup redis-server > /var/log/redis.log 2>&1 &
-    cd -
-}
-
-function start_rabbitmq() {
-    cd /tmp
-    nohup rabbitmq-server > /var/log/rabbitmq.log 2>&1 &
+    nohup valkey-server > /var/log/valkey.log 2>&1 &
     cd -
 }
 
 gosu postgres pg_ctl -D "$PGDATA" start && ln -s /tmp/.s.PGSQL.5432 /var/run/postgresql/.s.PGSQL.5432
-start_redis
-start_rabbitmq
+start_valkey
 python3 manage.py check
-coverage run --source=api,scheduler manage.py test --settings=api.settings.testing --noinput api scheduler.tests --exclude-tag=auth
-coverage report -m
+coverage run --source=api,scheduler manage.py test --settings=api.settings.testing --noinput api scheduler.tests --exclude-tag=auth --parallel="${TEST_PARALLEL:-8}"
diff --git a/rootfs/bin/upload-coverage b/rootfs/bin/upload-coverage
index 53433f3d0..492452b18 100755
--- a/rootfs/bin/upload-coverage
+++ b/rootfs/bin/upload-coverage
@@ -6,4 +6,8 @@
 # fail hard and fast even on pipelines
 set -eou pipefail
 
-codecov --required
+coverage report -m > coverage.txt
+
+if [[ -n $CODECOV_TOKEN ]]; then
+  codecov upload-process --plugin=noop -t "$CODECOV_TOKEN"
+fi
diff --git a/rootfs/dev_requirements.txt b/rootfs/dev_requirements.txt
index 058b80601..1f133f1de 100644
--- a/rootfs/dev_requirements.txt
+++ b/rootfs/dev_requirements.txt
@@ -1,13 +1,13 @@
 # test module
 # test
 # Run "make test-unit" for the % of code exercised during tests
-coverage==7.2.5
+coverage==7.10.6
 
 # Run "make test-style" to check python syntax and style
-flake8==6.0.0
-
-# code coverage report at https://codecov.io/github/drycc/controller
-codecov==2.1.13
+flake8==7.3.0
 
 # mock out python-requests, mostly k8s
-requests-mock==1.10.0
+requests-mock==1.12.1
+
+# tracebacks
+tblib==3.1.0
diff --git a/rootfs/drycc/gunicorn/config.py b/rootfs/drycc/gunicorn/config.py
index 77e7c20d4..9a980075c 100644
--- a/rootfs/drycc/gunicorn/config.py
+++ b/rootfs/drycc/gunicorn/config.py
@@ -1,21 +1,29 @@
 import os
-from os.path import dirname, realpath
+from os.path import dirname, realpath, exists
 
 import faulthandler
 faulthandler.enable()
 
-# If there is a mutating admission webhook configuration, start webhook
-if os.path.exists("/etc/controller/webhook/cert"):
+# If a certificate exists, set the certificate and bind to port 8443.
+CERT_KEY_PATH = os.environ.get(
+    'DRYCC_CERT_KEY_PATH', '/etc/controller/cert/key')
+CERT_TLS_KEY_PATH = os.environ.get(
+    'DRYCC_CERT_TLS_KEY_PATH', '/etc/controller/cert/tls.key')
+CERT_TLS_CRT_PATH = os.environ.get(
+    'DRYCC_CERT_TLS_CRT_PATH', '/etc/controller/cert/tls.crt')
+if exists(CERT_KEY_PATH) and exists(CERT_TLS_KEY_PATH) and exists(CERT_TLS_CRT_PATH):
     bind = '0.0.0.0:8443'
-    keyfile = "/etc/controller/webhook/cert/tls.key"
-    certfile = "/etc/controller/webhook/cert/tls.crt"
+    keyfile = CERT_TLS_KEY_PATH
+    certfile = CERT_TLS_CRT_PATH
+    reload_extra_files = [CERT_KEY_PATH, CERT_TLS_KEY_PATH, CERT_TLS_CRT_PATH]
 else:
     bind = '0.0.0.0:8000'
 
-workers = int(os.environ.get('GUNICORN_WORKERS', 2))
+workers = int(os.environ.get('GUNICORN_WORKERS', 4))
+timeout = int(os.environ.get('GUNICORN_TIMEOUT', 30))
+keepalive = int(os.environ.get('GUNICORN_KEEPALIVE', 60))
 worker_class = "uvicorn.workers.UvicornWorker"
 pythonpath = dirname(dirname(dirname(realpath(__file__))))
-timeout = 1200
 pidfile = '/tmp/gunicorn.pid'
 logger_class = 'drycc.gunicorn.logging.Logging'
 loglevel = 'info'
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
index 324cb5549..0c80ff269 100644
--- a/rootfs/requirements.txt
+++ b/rootfs/requirements.txt
@@ -1,32 +1,25 @@
 # Drycc controller requirements
+inotify==0.2.12
 backoff==2.2.1
-django==4.2.7
-channels==4.0.0
-aiohttp==3.9.0
-django-cors-headers==4.1.0
-django-guardian==2.4.0
-djangorestframework==3.14.0
-docker==6.1.3
-gunicorn==20.1.0
-uvicorn[standard]==0.22.0
-asgiref==3.7.2
-idna==3.4
-jmespath==1.0.1
-jsonschema==4.18.0
+django==5.2.9
+channels==4.3.1
+aiohttp==3.13.3
+django-cors-headers==4.9.0
+djangorestframework==3.16.1
+docker==7.1.0
+gunicorn==23.0.0
+uvicorn[standard]==0.38.0
+asgiref==3.10.0
+idna==3.11
+jsonschema==4.25.1
 morph==0.1.5
-packaging==23.1
-pyasn1==0.5.0
-psycopg[binary]==3.1.9
-pyOpenSSL==23.2.0
-ndg-httpsclient==0.5.1
-pytz==2023.3
-requests==2.31.0
+packaging==25.0
+psycopg[binary]==3.2.12
+pytz==2025.2
+requests==2.33.0
 requests-toolbelt==1.0.0
-celery==5.3.1
-hiredis==2.2.3
-django_redis==5.3.0
-dj-database-url==2.0.0
-social-auth-app-django==5.2.0
-python-jose==3.3.0
-Authlib==1.2.1
-kubernetes==26.1.0
+celery[redis]==5.5.3
+hiredis==3.3.0
+dj-database-url==3.0.1
+social-auth-app-django==5.6.0
+kubernetes==34.1.0
diff --git a/rootfs/scheduler/__init__.py b/rootfs/scheduler/__init__.py
index 6bfa035a0..223364a42 100644
--- a/rootfs/scheduler/__init__.py
+++ b/rootfs/scheduler/__init__.py
@@ -1,22 +1,19 @@
 from collections import OrderedDict
-from datetime import datetime
+from datetime import datetime, timezone
 import logging
 from packaging.version import Version, parse
 import requests
 import requests.exceptions
 from requests_toolbelt import user_agent
 import re
-import time
 from urllib.parse import urljoin
 
-from api import __version__ as drycc_version
-from scheduler.exceptions import KubeException, KubeHTTPException   # noqa
-from scheduler.states import PodState
+from api import utils, __version__ as drycc_version
+from scheduler.exceptions import KubeException, KubeHTTPException
 
 
 logger = logging.getLogger(__name__)
 session = None
-resource_mapping = OrderedDict()
 
 
 def get_k8s_session(k8s_api_verify_tls):
@@ -42,12 +39,13 @@ class KubeHTTPClient(object):
     api_prefix = 'api'
     # ISO-8601 which is used by kubernetes
     DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%SZ'
+    resource_mapping = OrderedDict()
 
-    def __init__(self, url, k8s_api_verify_tls=True):
-        global resource_mapping
+    def __init__(self, url, k8s_api_verify_tls=True, metadata=None):
         self.url = url
         self.k8s_api_verify_tls = k8s_api_verify_tls
         self.session = get_k8s_session(self.k8s_api_verify_tls)
+        self.metadata = metadata
 
         # map the various k8s Resources to an internal property
         from scheduler.resources import Resource  # lazy load
@@ -55,33 +53,32 @@ def __init__(self, url, k8s_api_verify_tls=True):
             name = str(res.__name__).lower()  # singular
             component = name + 's'  # make plural
             # check if component has already been processed
-            if component in resource_mapping:
+            if component in self.resource_mapping:
                 continue
 
             # get past recursion problems in case of self reference
-            resource_mapping[component] = ''
-            resource_mapping[component] = res(self.url, self.k8s_api_verify_tls)
+            self.resource_mapping[component] = ''
+            self.resource_mapping[component] = res(self.url, self.k8s_api_verify_tls)
             # map singular Resource name to the plural one
-            resource_mapping[name] = component
+            self.resource_mapping[name] = component
             if res.short_name is not None:
                 # map short name to long name so a resource can be named rs
                 # but have the main object live at replicasets
-                resource_mapping[str(res.short_name).lower()] = component
+                self.resource_mapping[str(res.short_name).lower()] = component
 
     def api(self, tmpl, *args):
         """Return a fully-qualified Kubernetes API URL from a string template with args."""
         return "/{}/{}".format(self.api_prefix, self.api_version) + tmpl.format(*args)
 
     def __getattr__(self, name):
-        global resource_mapping
-        if name in resource_mapping:
+        if name in self.resource_mapping:
             # resolve to final name if needed
-            component = resource_mapping[name]
+            component = self.resource_mapping[name]
             if type(component) is not str:
                 # already a component object
                 return component
 
-            return resource_mapping[component]
+            return self.resource_mapping[component]
 
         return object.__getattribute__(self, name)
 
@@ -98,7 +95,7 @@ def version(self):
 
     @staticmethod
     def parse_date(date):
-        return datetime.strptime(date, KubeHTTPClient.DATETIME_FORMAT)
+        return datetime.strptime(date, KubeHTTPClient.DATETIME_FORMAT).replace(tzinfo=timezone.utc)
 
     @staticmethod
     def unhealthy(status_code):
@@ -144,7 +141,7 @@ def query_params(labels=None, fields=None, resource_version=None, pretty=False):
         return query
 
     @staticmethod
-    def log(namespace, message, level='INFO'):
+    def log(namespace, message, level=logging.INFO):
         """Logs a message in the context of this application.
 
         This prefixes log messages with a namespace "tag".
@@ -152,8 +149,8 @@ def log(namespace, message, level='INFO'):
         sort like releasing or scaling, will be considered as "belonging" to the application
         instead of the controller and will be handled accordingly.
         """
-        lvl = getattr(logging, level.upper()) if hasattr(logging, level.upper()) else logging.INFO
-        logger.log(lvl, "[{}]: {}".format(namespace, message))
+        utils.send_app_log(namespace, message, level)
+        logger.log(level, "[{}]: {}".format(namespace, message))
 
     def http_head(self, path, **kwargs):
         """
@@ -194,6 +191,8 @@ def http_post(self, path, data=None, json=None, **kwargs):
         """
         try:
             url = urljoin(self.url, path)
+            if self.metadata is not None and "metadata" in data:
+                data["metadata"] = utils.dict_merge(data["metadata"], self.metadata)
             response = self.session.post(url, data=data, json=json, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # reraise as KubeException, but log stacktrace.
@@ -211,6 +210,8 @@ def http_put(self, path, data=None, **kwargs):
         """
         try:
             url = urljoin(self.url, path)
+            if self.metadata is not None and "metadata" in data:
+                data["metadata"] = utils.dict_merge(data["metadata"], self.metadata)
             response = self.session.put(url, data=data, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # reraise as KubeException, but log stacktrace.
@@ -233,6 +234,8 @@ def http_patch(self, path, data=None, **kwargs):
             # application/merge-patch+json,
             # application/apply-patch+yaml
             # self.session.headers["Content-Type"] = "application/json-patch+json"
+            if self.metadata is not None and "metadata" in data:
+                data["metadata"] = utils.dict_merge(data["metadata"], self.metadata)
             response = self.session.patch(url, data=data, **kwargs)
         except requests.exceptions.ConnectionError as err:
             # reraise as KubeException, but log stacktrace.
@@ -260,7 +263,7 @@ def http_delete(self, path, **kwargs):
 
         return response
 
-    def deploy(self, namespace, name, image, entrypoint, command, **kwargs):  # noqa
+    def deploy(self, namespace, name, image, command, args, **kwargs):
         """Deploy Deployment depending on what's requested"""
         app_type = kwargs.get('app_type')
         version = kwargs.get('version')
@@ -299,13 +302,13 @@ def deploy(self, namespace, name, image, entrypoint, command, **kwargs):  # noqa
         except KubeException:
             # create the initial deployment object (and the first revision)
             self.deployment.create(
-                namespace, name, image, entrypoint, command, spec_annotations, **kwargs
+                namespace, name, image, command, args, spec_annotations, **kwargs
             )
         else:
             try:
                 # kick off a new revision of the deployment
                 self.deployment.update(
-                    namespace, name, image, entrypoint, command, spec_annotations, **kwargs
+                    namespace, name, image, command, args, spec_annotations, **kwargs
                 )
             except KubeException as e:
                 raise KubeException(
@@ -313,7 +316,7 @@ def deploy(self, namespace, name, image, entrypoint, command, **kwargs):  # noqa
                     "Additional information:\n{}".format(version, namespace, app_type, str(e))
                 ) from e
 
-    def scale(self, namespace, name, image, entrypoint, command, **kwargs):
+    def scale(self, namespace, name, image, command, args, **kwargs):
         """Scale Deployment"""
         try:
             self.deployment.get(namespace, name)
@@ -323,7 +326,7 @@ def scale(self, namespace, name, image, entrypoint, command, **kwargs):
                 try:
                     spec_annotations = {}
                     self.deployment.create(
-                        namespace, name, image, entrypoint, command, spec_annotations, **kwargs
+                        namespace, name, image, command, args, spec_annotations, **kwargs
                     )
                 except KubeException:
                     # see if the deployment got created
@@ -338,60 +341,5 @@ def scale(self, namespace, name, image, entrypoint, command, **kwargs):
         # let the scale failure bubble up
         self.deployment.scale(namespace, name, **kwargs)
 
-    def run(self, namespace, name, image, entrypoint, command, **kwargs):
-        """Run a one-off command."""
-        self.log(namespace, 'run {}, img {}, entrypoint {}, cmd "{}"'.format(
-            name, image, entrypoint, command)
-        )
-
-        # run pods never restart
-        kwargs['restart_policy'] = 'Never'
-        kwargs['command'] = entrypoint
-        kwargs['args'] = command
-
-        self.pod.create(namespace, name, image, **kwargs)
-
-        try:
-            # give pod 20 minutes to execute (after it got into ready state)
-            # this is a fairly arbitrary limit but the gunicorn worker / LBs
-            # will make this timeout around 20 anyway.
-            # TODO: Revisit in the future so it can run longer
-            state = 'up'  # pod is still running
-            waited = 0
-            timeout = 1200  # 20 minutes
-            while (state == 'up' and waited < timeout):
-                pod = self.pod.get(namespace, name).json()
-                state = str(self.pod.state(pod))
-                # default data
-                exit_code = 0
-
-                waited += 1
-                time.sleep(1)
-
-            if state == 'down':  # run finished successfully
-                exit_code = 0  # successful run
-            elif state == 'crashed':  # run failed
-                pod_state = pod['status']['containerStatuses'][0]['state']
-                exit_code = pod_state['terminated']['exitCode']
-
-            # timed out!
-            if waited == timeout:
-                raise KubeException('Timed out (20 mins) while running')
-
-            # check if it is possible to get logs
-            state = self.pod.state(self.pod.get(namespace, name).json())
-            # States below up do not have logs
-            if not isinstance(state, PodState) or state < PodState.up:
-                return exit_code, 'Could not get logs. Pod is in state {}'.format(str(state))
-
-            # grab log information
-            log = self.pod.logs(namespace, name)
-            log.encoding = 'utf-8'  # defaults to "ISO-8859-1" otherwise...
-
-            return exit_code, log.text
-        finally:
-            # cleanup
-            self.pod.delete(namespace, name)
-
 
 SchedulerClient = KubeHTTPClient
diff --git a/rootfs/scheduler/mock.py b/rootfs/scheduler/mock.py
index fd9c2b837..7d016475e 100644
--- a/rootfs/scheduler/mock.py
+++ b/rootfs/scheduler/mock.py
@@ -1,5 +1,5 @@
 import copy
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import json
 import random
 import re
@@ -18,6 +18,8 @@
 from django.conf import settings
 from django.core.cache import cache
 
+from api import utils
+
 import logging
 logger = logging.getLogger(__name__)
 
@@ -49,7 +51,7 @@ def wrapped(*args, **kargs):
         return wrapped
 
     def __enter__(self):
-        if not type(self.key) == str and self.key == '':
+        if type(self.key) is not str and self.key == '':
             raise RuntimeError("Key not specified!")
 
         if not self.acquire(self.block):
@@ -83,7 +85,8 @@ def _acquire(self):
     'horizontalpodautoscalers', 'scale', 'resourcequotas', 'ingresses',
     'persistentvolumeclaims', 'serviceinstances', 'servicebindings',
     'limitranges', 'gateways', 'httproutes', 'tcproutes', 'udproutes',
-    'issuers', 'certificates'
+    'issuers', 'certificates', 'certificaterequests', 'jobs', 'clusterserviceclasses',
+    'statefulsets', 'daemonsets',
 ]
 
 
@@ -219,7 +222,7 @@ def pod_state_transitions(pod_url=None):
     pods = cache.get('pods_states', {})
     # Is there a new pod?
     if pod_url:
-        state_time = datetime.utcnow() + timedelta(seconds=jit())
+        state_time = datetime.now(timezone.utc) + timedelta(seconds=jit())
         pods[pod_url] = state_time
 
         # Initial state is Pending
@@ -230,7 +233,7 @@ def pod_state_transitions(pod_url=None):
 
     # Loops through all the pods to see if next phase needs to be done
     for pod_url, state_time in pods.items():
-        if datetime.utcnow() < state_time:
+        if datetime.now(timezone.utc) < state_time:
             # it is not time yet!
             continue
 
@@ -271,7 +274,7 @@ def cleanup_pods():
     """Can be called during any sort of access, it will cleanup pods as needed"""
     pods = cache.get('cleanup_pods', {})
     for pod, timestamp in pods.copy().items():
-        if timestamp > datetime.utcnow():
+        if timestamp > datetime.now(timezone.utc):
             continue
 
         del pods[pod]
@@ -288,13 +291,13 @@ def add_cleanup_pod(url):
 
     # save
     pods = cache.get('cleanup_pods', {})
-    pods[url] = (datetime.utcnow() + timedelta(seconds=grace))
+    pods[url] = (datetime.now(timezone.utc) + timedelta(seconds=grace))
     cache.set('cleanup_pods', pods)
 
     # add grace period timestamp
     pod = cache.get(url)
     grace = settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS
-    pd = datetime.utcnow() + timedelta(seconds=grace)
+    pd = datetime.now(timezone.utc) + timedelta(seconds=grace)
     timestamp = str(pd.strftime(MockSchedulerClient.DATETIME_FORMAT))
     pod['metadata']['deletionTimestamp'] = timestamp
     cache.set(url, pod)
@@ -350,7 +353,7 @@ def create_pods(url, labels, base, new_pods):
     for _ in range(new_pods):
         data = base.copy()
         # creation time
-        timestamp = str(datetime.utcnow().strftime(MockSchedulerClient.DATETIME_FORMAT))
+        timestamp = str(datetime.now(timezone.utc).strftime(MockSchedulerClient.DATETIME_FORMAT))
         data['metadata']['creationTimestamp'] = timestamp
         data['metadata']['uid'] = str(uuid.uuid4())
 
@@ -358,7 +361,7 @@ def create_pods(url, labels, base, new_pods):
         if 'generateName' in data['metadata']:
             data['metadata']['name'] = data['metadata']['generateName'] + pod_name()
 
-        timestamp = str(datetime.utcnow().strftime(MockSchedulerClient.DATETIME_FORMAT))
+        timestamp = str(datetime.now(timezone.utc).strftime(MockSchedulerClient.DATETIME_FORMAT))
         data['status'] = {
             'startTime': timestamp,
             'conditions': [
@@ -368,6 +371,7 @@ def create_pods(url, labels, base, new_pods):
             'containerStatuses': [
                 {
                     'name': '{}-{}'.format(labels['app'], labels['type']),
+                    'image': '127.0.0.1/{}-{}'.format(labels['app'], labels['type']),
                     # TODO ready can be True / False (boolean)
                     'ready': True,
                     # TODO can be running / terminated / waiting
@@ -375,7 +379,17 @@ def create_pods(url, labels, base, new_pods):
                         'running': {
                             'startedAt': timestamp
                         }
-                    }
+                    },
+                    'lastState': {
+                        "terminated": {
+                            "containerID": f"cri-o://{uuid.uuid4().hex * 2}",
+                            "exitCode": 1,
+                            "finishedAt": timestamp,
+                            "reason": "Error",
+                            "startedAt": timestamp
+                        }
+                    },
+                    'restartCount': 0,
                 }
             ],
         }
@@ -384,9 +398,7 @@ def create_pods(url, labels, base, new_pods):
         pod_key = url
         if cache_key(data['metadata']['name']) not in url:
             pod_key = cache_key(url + '_' + data['metadata']['name'])
-
         add_cache_item(pod_key, 'pods', data)
-
         # set up a fake log for the pod
         log = "I did stuff today"
         pod_log_key = pod_key + '_log'
@@ -606,7 +618,6 @@ def fetch_single(request, context):
         context.status_code = 404
         context.reason = 'Not Found'
         return {}
-
     return data
 
 
@@ -616,10 +627,6 @@ def fetch_all(request, context):
     filters = prepare_query_filters(url.query)
     cache_path = cache_key(request.path)
     data = filter_data(filters, cache_path)
-    # HACK(bacongobbler): in kubernetes v1.5, fetching a list of resources returns a NoneType
-    # instead of an empty list.
-    if not data:
-        data = None
     return {'items': data}
 
 
@@ -691,7 +698,7 @@ def post(request, context):
         return {}
 
     # fill in generic data
-    timestamp = str(datetime.utcnow().strftime(MockSchedulerClient.DATETIME_FORMAT))
+    timestamp = str(datetime.now(timezone.utc).strftime(MockSchedulerClient.DATETIME_FORMAT))
     data['metadata']['creationTimestamp'] = timestamp
     data['metadata']['resourceVersion'] = 1
     data['metadata']['uid'] = str(uuid.uuid4())
@@ -722,24 +729,76 @@ def post(request, context):
     # Handle RC / RS / Deployments
     if resource_type in ['replicationcontrollers', 'replicasets', 'deployments']:
         data['status'] = {
-            'observedGeneration': 1
+            'observedGeneration': 1,
+            'readyReplicas': 1,
+            'replicas': 1
         }
         data['metadata']['generation'] = 1
 
         if resource_type in ['replicationcontrollers', 'replicasets']:
             upsert_pods(data, url)
-        elif resource_type == 'deployments':
+        elif resource_type in 'deployments':
             manage_replicasets(data, url)
 
     # drycc run is the only thing that creates pods directly
+    change_cache(url, resource_type, data)
+
+    context.status_code = 201
+    context.reason = 'Created'
+    return data
+
+
+def change_cache(url, resource_type, data):
     if resource_type == 'pods':
         create_pods(url, data['metadata']['labels'], data, 1)
+    elif resource_type == 'jobs':
+        create_jobs(url, resource_type, data)
+    elif resource_type in ('issuers', 'certificates', 'certificaterequests'):
+        add_cert_manager_item(url, resource_type, data)
     else:
         add_cache_item(url, resource_type, data)
 
-    context.status_code = 201
-    context.reason = 'Created'
-    return data
+
+def add_cert_manager_item(url, resource_type, data):
+    conditions = [{
+        "lastTransitionTime": datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "message": f"mock {resource_type} message ok",
+        "reason": "reason",
+        "status": "True",
+        "type": "Ready",
+    }]
+    data["status"] = {"conditions": conditions}
+    add_cache_item(url, resource_type, data)
+    if resource_type == "certificates":
+        name = "%s-auto-tls-1" % data["metadata"]["name"]
+        namespace = data["metadata"]["namespace"]
+        labels = data["metadata"].get("labels", {})
+        url = f"apis_cert_manager.io_v1_namespaces_{namespace}_certificaterequests_{name}"
+        url = url.replace("-", "_")
+        request = {
+            "apiVersion": "cert-manager.io/v1",
+            "kind": "CertificateRequest",
+            "metadata": {"name": name, "namespace": namespace, "labels": labels},
+            "spec": {
+                "extra": {},
+                "groups": ["system:serviceaccounts"],
+                "issuerRef": {"kind": "Issuer", "name": "%s-issuer" % name},
+                "request": "test",
+                "uid": "1f656b19-d19d-4233-bfe7-f0cf81b65126",
+                "username": "system:serviceaccount:cert-manager:cert-manager"
+            },
+            "status": {"conditions": conditions}
+        }
+        add_cache_item(url, "certificaterequests", request)
+
+
+def create_jobs(url, resource_type, data):
+    add_cache_item(url, resource_type, data)
+    randstr = ''.join(random.choices(string.ascii_lowercase, k=5))
+    namespace = data['metadata']['namespace']
+    pod_data = data['spec']['template']
+    pod_url = f"api_v1_namespaces_{namespace}_pods_{namespace}_run_{int(time.time())}_{randstr}"
+    create_pods(pod_url, pod_data['metadata']['labels'], pod_data, 1)
 
 
 def put(request, context):
@@ -805,7 +864,7 @@ def put(request, context):
             manage_replicasets(data, url)
     else:
         # Update the individual resource
-        cache.set(url, data, None)
+        change_cache(url, resource_type, data)
 
     context.status_code = 200
     context.reason = 'OK'
@@ -846,7 +905,7 @@ def patch(request, context):  # noqa: C901
     data = request.json()
 
     # merge new data into old but keep labels separate in case they changed
-    if 'labels' in data['metadata']:
+    if 'labels' in data.get('metadata', {}):
         labels = data['metadata'].pop('labels')
         item['metadata'].update(data['metadata'])
         data['metadata'] = item['metadata']
@@ -879,9 +938,13 @@ def patch(request, context):  # noqa: C901
     elif resource_type in ['gateways']:
         data['status'] = item['status']
         cache.set(url, data, None)
+    elif resource_type in ['daemonsets', 'statefulsets']:
+        # item data merge-patch
+        data = utils.dict_merge(item, data)
+        cache.set(url, data, None)
     else:
         # Update the individual resource
-        cache.set(url, data, None)
+        change_cache(url, resource_type, data)
 
     context.status_code = 200
     context.reason = 'OK'
@@ -938,7 +1001,8 @@ def add_cache_item(url, resource_type, data):
 
     # Keep track of what resources exist under other resources (mostly namespace)
     # sneaky way of getting data up to the resource type without too much magic
-    cache_url = ''.join(url.partition(resource_type)[0:2])
+    index = url.rfind(resource_type)
+    cache_url = url[:index + len(resource_type)] if index != -1 else url
     items = cache.get(cache_url, [])
     if url not in items:
         items.append(url)
@@ -1014,8 +1078,8 @@ def session():
 
 
 class MockSchedulerClient(KubeHTTPClient):
-    def __init__(self, url, k8s_api_verify_tls=True):
-        super().__init__(url)
+    def __init__(self, url, k8s_api_verify_tls=True, metadata=None):
+        super().__init__(url, k8s_api_verify_tls, metadata)
 
         # set version data
         cache.set('version', {'major': '1', 'minor': '3'}, None)
@@ -1036,20 +1100,27 @@ def __init__(self, url, k8s_api_verify_tls=True):
             self.secret.create('drycc', 'objectstorage-keyfile', secrets)
 
         try:
-            self.secret.get('drycc', 'registry-secret')
+            self.secret.get('drycc', 'controller-creds')
         except KubeHTTPException:
             secrets = {
-                'username': 'test',
-                'password': 'test',
-                'hostname': ''
+                'registry-host': '',
+                'registry-username': 'test',
+                'registry-password': 'test',
             }
-            self.secret.create('drycc', 'registry-secret', secrets)
+            self.secret.create('drycc', 'controller-creds', secrets)
 
         try:
             self.ns.get('duplicate')
         except KubeHTTPException:
             self.ns.create('duplicate')
-
+        # node
+        self._setup_node()
+        # daemonset
+        self._setup_daemonset()
+        # statefulset
+        self._setup_statefulset()
+
+    def _setup_node(self):
         try:
             self.node.get('172.17.8.100')
         except KubeHTTPException:
@@ -1126,12 +1197,146 @@ def __init__(self, url, k8s_api_verify_tls=True):
                     }
                 }
             }
-
             name = 'api_v1_nodes_172.17.8.100'
             cache.set(name, data, None)
             cache.set('api_v1_nodes', [name], None)
-        except Exception as e:
-            logger.critical(e)
+
+    def _setup_daemonset(self):
+        data = {
+            "apiVersion": "apps/v1",
+            "kind": "DaemonSet",
+            "metadata": {
+                "name": "drycc",
+                "namespace": "drycc",
+                "labels": {
+                    "app.kubernetes.io/managed-by": "Helm",
+                    "app.kubernetes.io/name": "drycc"
+                }
+            },
+            "spec": {
+                "selector": {
+                    "matchLabels": {
+                        "app.kubernetes.io/name": "drycc"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app.kubernetes.io/managed-by": "Helm",
+                            "app.kubernetes.io/name": "drycc"
+                        }
+                    },
+                    "spec": {
+                        "affinity": {
+                            "nodeAffinity": {
+                                "requiredDuringSchedulingIgnoredDuringExecution": {
+                                    "nodeSelectorTerms": [
+                                        {
+                                            "matchExpressions": [{
+                                                "key": "kubernetes.io/hostname",
+                                                "operator": "In",
+                                                "values": [
+                                                    "server-01",
+                                                    "server-02",
+                                                    "server-03"
+                                                ]}
+                                            ]
+                                        }
+                                    ]
+                                }
+                            }
+                        },
+                        "containers": [{
+                            "image": "registry.drycc.cc/drycc/drycc",
+                            "imagePullPolicy": "IfNotPresent",
+                            "name": "drycc"
+                        }]
+                    }
+                }
+            },
+            "status": {
+                "currentNumberScheduled": 3,
+                "desiredNumberScheduled": 3,
+                "numberAvailable": 3,
+                "numberMisscheduled": 0,
+                "numberReady": 3,
+                "observedGeneration": 2,
+                "updatedNumberScheduled": 3
+            }
+        }
+        name = 'apis_apps_v1_namespaces_drycc_daemonsets_drycc'
+        cache.set(name, data, None)
+        cache.set('apis_apps_v1_namespaces_drycc_daemonsets', [name], None)
+
+    def _setup_statefulset(self):
+        data = {
+            "apiVersion": "apps/v1",
+            "kind": "StatefulSet",
+            "metadata": {
+                "labels": {
+                    "app.kubernetes.io/managed-by": "Helm",
+                    "app.kubernetes.io/name": "drycc"
+                },
+                "name": "drycc",
+                "namespace": "drycc"
+            },
+            "spec": {
+                "persistentVolumeClaimRetentionPolicy": {
+                    "whenDeleted": "Delete",
+                    "whenScaled": "Retain"
+                },
+                "selector": {
+                    "matchLabels": {
+                        "app.kubernetes.io/name": "drycc"
+                    }
+                },
+                "template": {
+                    "metadata": {
+                        "labels": {
+                            "app.kubernetes.io/managed-by": "Helm",
+                            "app.kubernetes.io/name": "drycc"
+                        }
+                    },
+                    "spec": {
+                            "containers": [{
+                                "image": "registry.drycc.cc/drycc/drycc",
+                                "imagePullPolicy": "IfNotPresent",
+                                "name": "drycc"
+                            }]
+                        }
+                    },
+                "volumeClaimTemplates": [{
+                    "apiVersion": "v1",
+                    "kind": "PersistentVolumeClaim",
+                    "metadata": {
+                        "name": "work-data"
+                    },
+                    "spec": {
+                        "accessModes": [
+                            "ReadWriteOnce"
+                        ],
+                        "resources": {
+                            "requests": {
+                                "storage": "1Gi"
+                            }
+                        },
+                        "volumeMode": "Filesystem"
+                    }
+                }]
+            },
+            "status": {
+                "availableReplicas": 2,
+                "collisionCount": 0,
+                "currentReplicas": 2,
+                "observedGeneration": 4,
+                "readyReplicas": 2,
+                "replicas": 2,
+                "updatedReplicas": 2
+            }
+        }
+        name = 'apis_apps_v1_namespaces_drycc_statefulsets_drycc'
+        cache.set(name, data, None)
+        cache.set('apis_apps_v1_namespaces_drycc_statefulsets', [name], None)
 
 
 scheduler.session = session()
diff --git a/rootfs/scheduler/resources/cert_manager.py b/rootfs/scheduler/resources/cert_manager.py
index a3c495392..eb622abc3 100644
--- a/rootfs/scheduler/resources/cert_manager.py
+++ b/rootfs/scheduler/resources/cert_manager.py
@@ -50,7 +50,7 @@ def manifest(self, namespace, name, **kwargs):
         }
         if "version" in kwargs:
             data["metadata"]["resourceVersion"] = kwargs.get("version")
-        if "key_id" in kwargs and "key_secret" in kwargs:
+        if kwargs.get("kwargs") and kwargs.get("key_secret"):
             data["spec"]["acme"]["externalAccountBinding"] = {
                 "keyID": kwargs["key_id"],
                 "keySecretRef": {
@@ -111,11 +111,8 @@ def manifest(api_version, namespace, name, hosts, version=None):
                 "namespace": namespace
             },
             "spec": {
-                "secretName": "%s-certificate-auto" % name,
-                "issuerRef": {
-                    "name": "drycc-cluster-issuer",
-                    "kind": "ClusterIssuer"
-                },
+                "secretName": name,
+                "issuerRef": {"name": namespace, "kind": "Issuer"},
                 "dnsNames": hosts
             }
         }
@@ -123,7 +120,7 @@ def manifest(api_version, namespace, name, hosts, version=None):
             data["metadata"]["resourceVersion"] = version
         return data
 
-    def get(self, namespace, name=None, **kwargs):
+    def get(self, namespace, name=None, ignore_exception=True, **kwargs):
         """
         Fetch a single certificate or a list of certificates
         """
@@ -134,7 +131,7 @@ def get(self, namespace, name=None, **kwargs):
             url = self.api('/namespaces/{}/certificates', namespace)
             message = 'get certificates'
         response = self.http_get(url)
-        if self.unhealthy(response.status_code):
+        if not ignore_exception and self.unhealthy(response.status_code):
             raise KubeHTTPException(response, message)
 
         return response
@@ -168,3 +165,24 @@ def delete(self, namespace, name, ignore_exception=False):
         if not ignore_exception and self.unhealthy(response.status_code):
             raise KubeHTTPException(response, 'delete certificate ' + name)
         return response
+
+
+class CertificateRequest(Resource):
+    api_version = 'cert-manager.io/v1'
+    api_prefix = 'apis'
+
+    def get(self, namespace, name=None, ignore_exception=True, **kwargs):
+        """
+        Fetch a single certificate or a list of certificaterequest
+        """
+        if name is not None:
+            url = self.api('/namespaces/{}/certificaterequests/{}', namespace, name)
+            message = 'get certificaterequest ' + name
+        else:
+            url = self.api('/namespaces/{}/certificaterequests', namespace)
+            message = 'get certificaterequests'
+        response = self.http_get(url)
+        if not ignore_exception and self.unhealthy(response.status_code):
+            raise KubeHTTPException(response, message)
+
+        return response
diff --git a/rootfs/scheduler/resources/daemonset.py b/rootfs/scheduler/resources/daemonset.py
new file mode 100644
index 000000000..9d7dfbd33
--- /dev/null
+++ b/rootfs/scheduler/resources/daemonset.py
@@ -0,0 +1,45 @@
+import json
+import logging
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class Daemonset(Resource):
+    api_prefix = 'apis'
+    api_version = 'apps/v1'
+
+    def get(self, namespace, name=None, ignore_exception=False, **kwargs):
+        """
+        Fetch a single Daemonsets or a list
+        """
+        url = '/namespaces/{}/daemonsets'
+        args = [namespace]
+        if name is not None:
+            args.append(name)
+            url += '/{}'
+            message = 'get Daemonset "{}" in Namespace "{}"'
+        else:
+            message = 'get Daemonsets in Namespace "{}"'
+
+        url = self.api(url, *args)
+        response = self.http_get(url, params=self.query_params(**kwargs))
+        if self.unhealthy(response.status_code) and not ignore_exception:
+            args.reverse()  # error msg is in reverse order
+            raise KubeHTTPException(response, message, *args)
+
+        return response
+
+    def patch(self, namespace, name, manifest, ignore_exception=False, **kwargs):
+        url = self.api("/namespaces/{}/daemonsets/{}", namespace, name)
+        response = self.http_patch(
+            url,
+            json=manifest,
+            headers={"Content-Type": "application/merge-patch+json"}
+        )
+
+        if self.unhealthy(response.status_code):
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'patch Daemonset "{}"', name)
+        return response
diff --git a/rootfs/scheduler/resources/deployment.py b/rootfs/scheduler/resources/deployment.py
index 9be4d58d3..59b0dca8e 100644
--- a/rootfs/scheduler/resources/deployment.py
+++ b/rootfs/scheduler/resources/deployment.py
@@ -1,6 +1,7 @@
-from datetime import datetime, timedelta
 import json
 import time
+import logging
+from datetime import datetime, timedelta, timezone
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeException, KubeHTTPException
 
@@ -9,7 +10,7 @@ class Deployment(Resource):
     api_prefix = 'apis'
     api_version = 'apps/v1'
 
-    def get(self, namespace, name=None, **kwargs):
+    def get(self, namespace, name=None, ignore_exception=False, **kwargs):
         """
         Fetch a single Deployment or a list
         """
@@ -24,13 +25,13 @@ def get(self, namespace, name=None, **kwargs):
 
         url = self.api(url, *args)
         response = self.http_get(url, params=self.query_params(**kwargs))
-        if self.unhealthy(response.status_code):
+        if self.unhealthy(response.status_code) and not ignore_exception:
             args.reverse()  # error msg is in reverse order
             raise KubeHTTPException(response, message, *args)
 
         return response
 
-    def manifest(self, namespace, name, image, entrypoint, command, spec_annotations, **kwargs):
+    def manifest(self, namespace, name, image, command, args, spec_annotations, **kwargs):
         replicas = kwargs.get('replicas', 0)
         batches = kwargs.get('deploy_batches', None)
         tags = kwargs.get('tags', {})
@@ -56,7 +57,7 @@ def manifest(self, namespace, name, image, entrypoint, command, spec_annotations
                 'replicas': replicas,
                 'selector': {
                     'matchLabels': labels
-                }
+                },
             }
         }
 
@@ -75,7 +76,6 @@ def manifest(self, namespace, name, image, entrypoint, command, spec_annotations
             manifest['spec']['rollbackTo'] = {'revision': str(revision)}
 
         # Add deployment strategy
-
         # see if application or global deploy batches are defined
         maxSurge = self._get_deploy_steps(batches, tags)
         # if replicas are higher than maxSurge then the old deployment is never scaled down
@@ -99,14 +99,15 @@ def manifest(self, namespace, name, image, entrypoint, command, spec_annotations
             manifest['spec']['revisionHistoryLimit'] = int(kwargs.get('deployment_revision_history_limit'))  # noqa
 
         # tell pod how to execute the process
-        kwargs['command'] = entrypoint
-        kwargs['args'] = command
+        kwargs['command'] = command
+        kwargs['args'] = args
 
         # pod manifest spec
         manifest['spec']['template'] = self.pod.manifest(namespace, name, image, **kwargs)
 
         # set the old deployment spec annotations on this deployment
         manifest['spec']['template']['metadata']['annotations'] = spec_annotations
+        manifest['spec']['template']['spec']['automountServiceAccountToken'] = False
         if annotations:
             manifest['spec']['template']['metadata']['annotations'].update(annotations)
         if "resource_version" in kwargs:
@@ -114,42 +115,47 @@ def manifest(self, namespace, name, image, entrypoint, command, spec_annotations
 
         return manifest
 
-    def create(self, namespace, name, image, entrypoint, command, spec_annotations, **kwargs):
+    def create(self, namespace, name, image, command, args,
+               spec_annotations, ignore_exception=False, **kwargs):
         manifest = self.manifest(namespace, name, image,
-                                 entrypoint, command, spec_annotations, **kwargs)
+                                 command, args, spec_annotations, **kwargs)
 
         url = self.api("/namespaces/{}/deployments", namespace)
         response = self.http_post(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
-            raise KubeHTTPException(
-                response,
-                'create Deployment "{}" in Namespace "{}"', name, namespace
-            )
-
-        self.wait_until_updated(namespace, name)
-        self.wait_until_ready(namespace, name, **kwargs)
-
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(
+                    response,
+                    'create Deployment "{}" in Namespace "{}"', name, namespace
+                )
+        else:
+            self.wait_until_updated(namespace, name)
+            self.wait_until_ready(namespace, name, **kwargs)
         return response
 
-    def update(self, namespace, name, image, entrypoint, command, spec_annotations, **kwargs):
+    def update(self, namespace, name, image, command, args,
+               spec_annotations, ignore_exception=False, **kwargs):
         manifest = self.manifest(namespace, name, image,
-                                 entrypoint, command, spec_annotations, **kwargs)
+                                 command, args, spec_annotations, **kwargs)
 
         url = self.api("/namespaces/{}/deployments/{}", namespace, name)
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
-            raise KubeHTTPException(response, 'update Deployment "{}"', name)
-
-        self.wait_until_updated(namespace, name)
-        self.wait_until_ready(namespace, name, **kwargs)
-
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'update Deployment "{}"', name)
+        else:
+            self.wait_until_updated(namespace, name)
+            self.wait_until_ready(namespace, name, **kwargs)
         return response
 
-    def patch(self, namespace, name, image, entrypoint, command, spec_annotations, **kwargs):
+    def patch(self, namespace, name, image, command, args,
+              spec_annotations, ignore_exception=False, **kwargs):
         manifest = self.manifest(namespace, name, image,
-                                 entrypoint, command, spec_annotations, **kwargs)
+                                 command, args, spec_annotations, **kwargs)
 
         url = self.api("/namespaces/{}/deployments/{}", namespace, name)
         response = self.http_patch(
@@ -159,12 +165,13 @@ def patch(self, namespace, name, image, entrypoint, command, spec_annotations, *
         )
 
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
-            raise KubeHTTPException(response, 'patch Deployment "{}"', name)
-
-        self.wait_until_updated(namespace, name)
-        self.wait_until_ready(namespace, name, **kwargs)
-
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'patch Deployment "{}"', name)
+        else:
+            self.wait_until_updated(namespace, name)
+            self.wait_until_ready(namespace, name, **kwargs)
         return response
 
     def delete(self, namespace, name, ignore_exception=False):
@@ -199,12 +206,12 @@ def scale(self, namespace, name, **kwargs):
             kwargs['previous_replicas'] = current
             self.wait_until_ready(namespace, name, **kwargs)
 
-    def restart(self, namespace, name):
+    def restart(self, namespace, name, ignore_exception=False):
         url = self.api(
             "/namespaces/{}/deployments/{}?fieldManager=kubectl-rollout&pretty=true",
             namespace, name
         )
-        restartedAt = "%sZ" % (timedelta(seconds=3) + datetime.utcnow()).isoformat("T")
+        restartedAt = "%sZ" % (timedelta(seconds=3) + datetime.now(timezone.utc)).isoformat("T")
         response = self.http_patch(
             url,
             data=json.dumps({
@@ -220,7 +227,7 @@ def restart(self, namespace, name):
             }),
             headers={"Content-Type": "application/merge-patch+json"},
         )
-        if self.unhealthy(response.status_code):
+        if self.unhealthy(response.status_code) and not ignore_exception:
             raise KubeHTTPException(
                 response,
                 'restart Deployment "{}" in Namespace "{}"', name, namespace
@@ -243,17 +250,17 @@ def in_progress(self, namespace, name, timeout, batches, replicas, tags):
         Returns 2 booleans, first one is for if the Deployment is in progress or not, second
         one is or if a rollback action is advised while leaving the rollback up to the caller
         """
-        self.log(namespace, 'Checking if Deployment {} is in progress'.format(name), level='DEBUG')  # noqa
+        self.log(namespace, 'Checking if Deployment {} is in progress'.format(name), level=logging.DEBUG)  # noqa
         try:
             ready, _ = self.are_replicas_ready(namespace, name)
             if ready:
                 # nothing more to do - False since it is not in progress
-                self.log(namespace, 'All replicas for Deployment {} are ready'.format(name), level='DEBUG')  # noqa
+                self.log(namespace, 'All replicas for Deployment {} are ready'.format(name), level=logging.DEBUG)  # noqa
                 return False, False
         except KubeHTTPException as e:
             # Deployment doesn't exist
             if e.response.status_code == 404:
-                self.log(namespace, 'Deployment {} does not exist yet'.format(name), level='DEBUG')  # noqa
+                self.log(namespace, 'Deployment {} does not exist yet'.format(name), level=logging.DEBUG)  # noqa
                 return False, False
 
         # get deployment information
@@ -274,7 +281,12 @@ def in_progress(self, namespace, name, timeout, batches, replicas, tags):
         try:
             timeout += self.pod._handle_pending_pods(namespace, labels)
         except KubeException as e:
-            self.log(namespace, 'Deployment {} had stalled due an error and will be rolled back. {}'.format(name, str(e)), level='DEBUG')  # noqa
+            self.log(
+                namespace,
+                'Deployment {} had stalled due an error and will be rolled back. {}'.format(
+                    name, str(e)),
+                level=logging.DEBUG
+            )
             return False, True
 
         # fetch the latest RS for Deployment and use the start time to compare to deploy timeout
@@ -289,8 +301,13 @@ def in_progress(self, namespace, name, timeout, batches, replicas, tags):
 
         # throw an exception if over TTL so error is bubbled up
         start = self.parse_date(replica['metadata']['creationTimestamp'])
-        if (start + timedelta(seconds=timeout)) < datetime.utcnow():
-            self.log(namespace, 'Deploy operation for Deployment {} in has expired. Rolling back to last good known release'.format(name), level='DEBUG')  # noqa
+        if (start + timedelta(seconds=timeout)) < datetime.now(timezone.utc):
+            self.log(
+                namespace,
+                'Deploy operation for Deployment {} in has expired. '
+                'Rolling back to last good known release'.format(name),
+                level=logging.DEBUG,
+            )
             return False, True
 
         try:
@@ -336,7 +353,7 @@ def wait_until_updated(self, namespace, name):
         More information is also available at:
         https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
-        self.log(namespace, "waiting for Deployment {} to get a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+        self.log(namespace, "waiting for Deployment {} to get a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
         for _ in range(30):
             try:
                 deploy = self.deployment.get(namespace, name).json()
@@ -344,7 +361,7 @@ def wait_until_updated(self, namespace, name):
                     'observedGeneration' in deploy['status'] and
                     deploy['status']['observedGeneration'] >= deploy['metadata']['generation']
                 ):
-                    self.log(namespace, "A newer generation was found for Deployment {}".format(name), 'DEBUG')  # noqa
+                    self.log(namespace, "A newer generation was found for Deployment {}".format(name), logging.DEBUG)  # noqa
                     break
 
                 time.sleep(1)
@@ -425,12 +442,12 @@ def _check_for_failed_events(self, namespace, labels):
         response = self.rs.get(namespace, labels=labels)
         data = response.json()
         fields = {
-            'involvedObject.kind': 'ReplicaSet',
-            'involvedObject.name': data['items'][0]['metadata']['name'],
-            'involvedObject.namespace': namespace,
-            'involvedObject.uid': data['items'][0]['metadata']['uid'],
+            'regarding.kind': 'ReplicaSet',
+            'regarding.name': data['items'][0]['metadata']['name'],
+            'regarding.namespace': namespace,
+            'regarding.uid': data['items'][0]['metadata']['uid'],
         }
-        events_list = self.ns.events(namespace, fields=fields).json()
+        events_list = self.ev.get(namespace, fields=fields).json()
         events = events_list.get('items', [])
         if events is not None and len(events) != 0:
             for event in events:
@@ -444,10 +461,11 @@ def _get_formatted_messages(events):
         """
         Format each event by string and join all events to one string
         """
-        message_format = 'Message:{message}, lastTimestamp:{lastTimestamp}, reason: {reason}, count: {count}'  # noqa
+        message_format = 'Message: {note}, creationTimestamp: {creation_timestamp}, reason: {reason}'  # noqa
         output = []
         for event in events:
-            output.append(message_format.format(**event))
+            creation_timestamp = event["metadata"]["creationTimestamp"]
+            output.append(message_format.format(creation_timestamp=creation_timestamp, **event))
         return '\n'.join(output)
 
     def _get_deploy_steps(self, batches, tags):
diff --git a/rootfs/scheduler/resources/events.py b/rootfs/scheduler/resources/events.py
index 3187defab..749098ac8 100644
--- a/rootfs/scheduler/resources/events.py
+++ b/rootfs/scheduler/resources/events.py
@@ -9,8 +9,9 @@
 class Events(Resource):
     """
     Events resource.
-    Warning! Used ONLY for testing purposes
     """
+    api_prefix = 'apis'
+    api_version = 'events.k8s.io/v1'
     short_name = 'ev'
 
     def create(self, namespace, name, message, **kwargs):
@@ -18,7 +19,6 @@ def create(self, namespace, name, message, **kwargs):
         data = {
             'kind': 'Event',
             'apiVersion': self.api_version,
-            'count': kwargs.get('count', 1),
             'metadata': {
                 'creationTimestamp': datetime.now().strftime(DATETIME_FORMAT),
                 'namespace': namespace,
@@ -26,15 +26,10 @@ def create(self, namespace, name, message, **kwargs):
                 'resourceVersion': kwargs.get('resourceVersion', ''),
                 'uid': str(uuid.uuid4()),
             },
-            'message': message,
+            'note': message,
             'type': kwargs.get('type', 'Normal'),
-            'firstTimestamp': datetime.now().strftime(DATETIME_FORMAT),
-            'lastTimestamp': datetime.now().strftime(DATETIME_FORMAT),
             'reason': kwargs.get('reason', ''),
-            'source': {
-                'component': kwargs.get('component', ''),
-            },
-            'involvedObject': kwargs.get('involvedObject', {})
+            'regarding': kwargs.get('regarding', {})
         }
 
         response = self.http_post(url, json=data)
@@ -42,3 +37,18 @@ def create(self, namespace, name, message, **kwargs):
             raise KubeHTTPException(response, 'create Event for namespace {}'.format(namespace))  # noqa
 
         return response
+
+    def get(self, namespace, ignore_exception=False, **kwargs):
+        """
+        Fetch Events
+        """
+        url = '/namespaces/{}/events'
+        args = [namespace]
+        message = 'get Events in Namespace "{}"'
+
+        url = self.api(url, *args)
+        response = self.http_get(url, params=self.query_params(**kwargs))
+        if self.unhealthy(response.status_code) and not ignore_exception:
+            args.reverse()  # error msg is in reverse order
+            raise KubeHTTPException(response, message, *args)
+        return response
diff --git a/rootfs/scheduler/resources/gateway.py b/rootfs/scheduler/resources/gateway.py
index 7f4710dd1..a51ff538a 100644
--- a/rootfs/scheduler/resources/gateway.py
+++ b/rootfs/scheduler/resources/gateway.py
@@ -13,9 +13,7 @@ def manifest(self, namespace, name, **kwargs):
             "metadata": {
                 "name": name,
                 "namespace": namespace,
-                "annotations": {
-                    "cert-manager.io/issuer": namespace
-                },
+                "annotations": kwargs.get("annotations", {}),
             },
             "spec": {
                 "gatewayClassName": kwargs.get("gateway_class", "default"),
@@ -61,8 +59,10 @@ def patch(self, namespace, name, ignore_exception=False, **kwargs):
             json=data,
             headers={"Content-Type": "application/merge-patch+json"}
         )
+        if response.status_code == 409:
+            return response
         if not ignore_exception and self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, "put gateway {}".format(namespace))
+            raise KubeHTTPException(response, "patch gateway {}".format(namespace))
         return response
 
     def delete(self, namespace, name, ignore_exception=True):
@@ -130,7 +130,7 @@ def patch(self, namespace, name, ignore_exception=False, **kwargs):
             headers={"Content-Type": "application/merge-patch+json"}
         )
         if not ignore_exception and self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, "put {} {}".format(self.kind.lower(), namespace))
+            raise KubeHTTPException(response, "patch {} {}".format(self.kind.lower(), namespace))
         return response
 
     def delete(self, namespace, name, ignore_exception=True):
diff --git a/rootfs/scheduler/resources/hpa.py b/rootfs/scheduler/resources/hpa.py
index f45d810f8..6523a0adf 100644
--- a/rootfs/scheduler/resources/hpa.py
+++ b/rootfs/scheduler/resources/hpa.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from packaging.version import parse
 
 from scheduler.resources import Resource
@@ -93,7 +94,7 @@ def create(self, namespace, name, app_type, target, **kwargs):
         url = self.api("/namespaces/{}/horizontalpodautoscalers", namespace)
         response = self.http_post(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(
                 response,
                 'create HorizontalPodAutoscaler "{}" in Namespace "{}"', name, namespace
@@ -111,7 +112,7 @@ def update(self, namespace, name, app_type, target, **kwargs):
         url = self.api("/namespaces/{}/horizontalpodautoscalers/{}", namespace, name)
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(response, 'update HorizontalPodAutoscaler "{}"', name)
 
         # optionally wait for HPA if requested
diff --git a/rootfs/scheduler/resources/job.py b/rootfs/scheduler/resources/job.py
new file mode 100644
index 000000000..077557f4a
--- /dev/null
+++ b/rootfs/scheduler/resources/job.py
@@ -0,0 +1,45 @@
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class Job(Resource):
+
+    api_prefix = 'apis'
+    api_version = 'batch/v1'
+
+    def manifest(self, namespace, name, image, command, args, **kwargs):
+        manifest = {
+            "apiVersion": "batch/v1",
+            "kind": "Job",
+            "metadata": {
+                "name": name,
+                'labels': {
+                    'app': namespace,
+                    'type': kwargs.get('app_type', 'run'),
+                    'heritage': 'drycc',
+                },
+            },
+            "spec": {
+                "backoffLimit": kwargs.get('backoff_limit', 0),
+                "activeDeadlineSeconds": kwargs.get('active_deadline_seconds', 3600),
+                "ttlSecondsAfterFinished": kwargs.get('ttl_seconds_after_finished', 86400),
+            }
+        }
+        # tell pod how to execute the process
+        kwargs['command'] = command
+        kwargs['args'] = args
+        # job labels
+        manifest['metadata']['labels'].update(kwargs.get('labels', {}))
+        # pod manifest spec
+        manifest['spec']['template'] = self.pod.manifest(namespace, name, image, **kwargs)
+        return manifest
+
+    def create(self, namespace, name, image, command,
+               args, ignore_exception=False, **kwargs):
+        data = self.manifest(namespace, name, image, command, args, **kwargs)
+        url = self.api("/namespaces/{}/jobs", namespace)
+        response = self.http_post(url, json=data)
+
+        if not ignore_exception and self.unhealthy(response.status_code):
+            raise KubeHTTPException(response, "create Job {}".format(namespace))
+        return response
diff --git a/rootfs/scheduler/resources/limits.py b/rootfs/scheduler/resources/limits.py
deleted file mode 100644
index 72e652bf2..000000000
--- a/rootfs/scheduler/resources/limits.py
+++ /dev/null
@@ -1,44 +0,0 @@
-from scheduler.exceptions import KubeHTTPException
-from scheduler.resources import Resource
-
-
-class LimitRanges(Resource):
-    short_name = 'limits'
-
-    def get(self, namespace_name, name):
-        """
-        Fetch a single LimitRanges
-        """
-        url = '/namespaces/{}/limitranges/{}'.format(namespace_name, name)
-        message = 'get LimitRanges {} for namespace {}'.format(name, namespace_name)
-        url = self.api(url)
-        response = self.http_get(url)
-        if self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, message)
-
-        return response
-
-    def create(self, namespace_name, name, **kwargs):
-        """
-        Create resource LimitRanges for namespace
-        """
-        url = self.api("/namespaces/{}/limitranges".format(namespace_name))
-        manifest = {
-            "apiVersion": self.api_version,
-            "kind": "LimitRange",
-            "metadata": {
-                "namespace": namespace_name,
-                "name": name,
-                'labels': {
-                    'app': namespace_name,
-                    'heritage': 'drycc'
-                },
-            },
-            'spec': kwargs.get('spec', {})
-        }
-        response = self.http_post(url, json=manifest)
-        if not response.status_code == 201:
-            raise KubeHTTPException(response,
-                                    "create LimitRanges {} for namespace {}".format(
-                                        name, namespace_name))
-        return response
diff --git a/rootfs/scheduler/resources/namespace.py b/rootfs/scheduler/resources/namespace.py
index ed420b9c5..120c9ee34 100644
--- a/rootfs/scheduler/resources/namespace.py
+++ b/rootfs/scheduler/resources/namespace.py
@@ -52,11 +52,3 @@ def delete(self, namespace):
             raise KubeHTTPException(response, 'delete Namespace "{}"', namespace)
 
         return response
-
-    def events(self, namespace, **kwargs):
-        url = self.api("/namespaces/{}/events", namespace)
-        response = self.http_get(url, params=self.query_params(**kwargs))
-        if self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, "get Events in Namespace {}", namespace)
-
-        return response
diff --git a/rootfs/scheduler/resources/pod.py b/rootfs/scheduler/resources/pod.py
index 6823d1064..94de5cd80 100644
--- a/rootfs/scheduler/resources/pod.py
+++ b/rootfs/scheduler/resources/pod.py
@@ -1,13 +1,15 @@
-from collections import defaultdict
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 import operator
 import os
 import time
+import math
 
 from scheduler.exceptions import KubeException, KubeHTTPException
 from scheduler.resources import Resource
 from scheduler.states import PodState
 
+DEFAULT_CONTAINER_PORT = 5000
+
 
 class Pod(Resource):
     short_name = 'po'
@@ -129,15 +131,11 @@ def affinity(cls, name):
 
     def manifest(self, namespace, name, image, **kwargs):
         app_type = kwargs.get('app_type')
-        volumes = kwargs.get('volumes')
 
         # labels that represent the pod(s)
-        labels = {
-            'app': namespace,
-            'version': kwargs.get('version'),
-            'type': app_type,
-            'heritage': 'drycc',
-        }
+        default_labels = {'app': namespace, 'version': kwargs.get('version'), 'heritage': 'drycc'}
+        labels = kwargs.get('labels', default_labels)
+        labels.update({'type': app_type})
 
         # create base pod structure
         manifest = {
@@ -150,7 +148,9 @@ def manifest(self, namespace, name, image, **kwargs):
               'annotations': kwargs.get('annotations', {}),
             },
             'spec': {
-                "affinity": self.affinity(name)
+                "affinity": self.affinity(name),
+                'nodeSelector': kwargs.get('node_selector', {}),
+                'securityContext': kwargs.get('pod_security_context', {}),
             }
         }
         # pod manifest spec
@@ -165,20 +165,12 @@ def manifest(self, namespace, name, image, **kwargs):
         spec['restartPolicy'] = kwargs.get('restart_policy', 'Always')
 
         # apply tags as needed to restrict pod to particular node(s)
-        spec['nodeSelector'] = kwargs.get('tags', {})
+        spec['nodeSelector'].update(kwargs.get('tags', {}))
 
         # How long until a pod is forcefully terminated. 30 is kubernetes default
-        spec['terminationGracePeriodSeconds'] = self._get_termination_grace_period(kwargs)  # noqa
-
-        if volumes:
-            exist_volumes = spec.get('volumes', [])
-            for volume in volumes:
-                exist_volumes.append(
-                    {"name": volume.get('name'),
-                     "persistentVolumeClaim": {
-                         "claimName": volume.get('claimName')
-                     }})
-            spec['volumes'] = exist_volumes
+        spec['terminationGracePeriodSeconds'] = kwargs.get('termination_grace_period_seconds', 30)
+        # set pod volumes
+        spec['volumes'] = kwargs.get('volumes', [])
         # create the base container
         container = {}
 
@@ -209,99 +201,61 @@ def _set_container(self, namespace, container_name, data, **kwargs):
         data['name'] = container_name
         # set the image to use
         data['image'] = kwargs.get('image')
+        # set resources to use
+        data["resources"] = kwargs.get("resources", {})
         # set the image pull policy for the above image
         data['imagePullPolicy'] = kwargs.get('image_pull_policy')
         # add in any volumes that need to be mounted into the container
-        data['volumeMounts'] = kwargs.get('volumeMounts', [])
+        data['volumeMounts'] = kwargs.get('volume_mounts', [])
         # set the security context to use
-        data["securityContext"] = kwargs.get('security_context', {})
+        data["securityContext"] = kwargs.get('container_security_context', {})
         # create env list if missing
-        if 'env' not in data:
-            data['env'] = []
+        data['env'] = data.get('env', [])
 
         if env:
             # map application configuration (env secret) to env vars
-            secret_name = "{}-{}-env".format(namespace, kwargs.get('version'))
+            secret_name = "{}-{}-{}-env".format(
+                namespace, kwargs.get('app_type'), kwargs.get('version'))
             for key in env.keys():
                 item = {
                     "name": key,
-                    "valueFrom": {
-                        "secretKeyRef": {
-                            "name": secret_name,
-                            # k8s doesn't allow _ so translate to -, see above
-                            "key": key.lower().replace('_', '-')
-                        }
-                    }
+                    "valueFrom": {"secretKeyRef": {
+                        "name": secret_name,
+                        # k8s doesn't allow _ so translate to -, see above
+                        "key": key.lower().replace('_', '-')
+                    }}
                 }
-
                 # add value to env hash. Overwrite hardcoded values if need be
                 match = next((k for k, e in enumerate(data["env"]) if e['name'] == key), None)
                 if match is not None:
                     data["env"][match] = item
                 else:
                     data["env"].append(item)
-
-        # Inject debugging if workflow is in debug mode
-        if os.environ.get("DRYCC_DEBUG", False):
-            data["env"].append({
-                "name": "DRYCC_DEBUG",
-                "value": "1"
-            })
-
+        # set container default env
+        self._set_container_default_env(data)
         # list sorted by dict key name
         data['env'].sort(key=operator.itemgetter('name'))
-
-        self._set_resources(data, kwargs)
-
+        data['lifecycle'] = kwargs.get('lifecycle', {})
         self._set_health_checks(data, env, **kwargs)
 
-        self._set_lifecycle_hooks(data, env, **kwargs)
-
-        self._set_volume_mounts(data, kwargs)
-
-    @staticmethod
-    def _set_volume_mounts(container, kwargs):
-        # set volume_mounts
-        volume_mounts = kwargs.get('volume_mounts', [])
-        exist_volume_mounts = container.get('volumeMounts', [])
-        for volume_mount in volume_mounts:
-            exist_volume_mounts.append(
-                {"name": volume_mount.get('name'),
-                 "mountPath": volume_mount.get('mount_path')})
-        container['volumeMounts'] = exist_volume_mounts
-
-    def _set_resources(self, container, kwargs):
-        """ Set CPU/memory resource management manifest """
-        app_type = kwargs.get("app_type")
-        mem = kwargs.get("memory", {}).get(app_type)
-        cpu = kwargs.get("cpu", {}).get(app_type)
-        resources = kwargs.get("resources", defaultdict(dict))
-        if mem or cpu:
-            if mem:
-                if "/" in mem:
-                    parts = mem.split("/")
-                    resources["requests"]["memory"] = self._get_memory(parts[0])
-                    resources["limits"]["memory"] = self._get_memory(parts[1])
-                else:
-                    resources["limits"]["memory"] = self._get_memory(mem)
-            if cpu:
-                # CPU needs to be defined as lower case
-                if "/" in cpu:
-                    parts = cpu.split("/")
-                    resources["requests"]["cpu"] = parts[0].lower()
-                    resources["limits"]["cpu"] = parts[1].lower()
-                else:
-                    resources["limits"]["cpu"] = cpu.lower()
-        container["resources"] = dict(resources)
-
-    @staticmethod
-    def _get_termination_grace_period(kwargs):
-        """ return termination grace period """
-        app_type = kwargs.get("app_type")
-        timeout_global = kwargs.get('pod_termination_grace_period_seconds', 30)
-        timeout_local = kwargs.get("pod_termination_grace_period_each", {}).get(app_type)
-
-        return timeout_global if timeout_local is None else int(timeout_local)
+    def _set_container_default_env(self, data):
+        # set fields env
+        fields = {"CONTAINER_IP": "status.podIP"}
+        for key, value in fields.items():
+            item = {
+                "name": key,
+                "valueFrom": {"fieldRef": {"fieldPath": value}}
+            }
+            # add value to env hash. Overwrite hardcoded values if need be
+            match = next((k for k, e in enumerate(data["env"]) if e['name'] == key), None)
+            if match is not None:
+                data["env"][match] = item
+            else:
+                data["env"].append(item)
+        # Inject debugging if workflow is in debug mode
+        if os.environ.get("DRYCC_DEBUG", False):
+            data["env"].append({"name": "DRYCC_DEBUG", "value": "1"})
+        return data
 
     @staticmethod
     def _get_memory(size):
@@ -317,49 +271,19 @@ def _get_memory(size):
     def _set_health_checks(self, container, env, **kwargs):
         healthchecks = kwargs.get('healthcheck', None)
         if healthchecks:
-            # check if a port is present. if not, auto-populate it
-            # TODO: rip this out when we stop supporting drycc config:set HEALTHCHECK_URL
+            # If the port is empty, set the default port
             if (
                 healthchecks.get('livenessProbe') is not None and
                 healthchecks['livenessProbe'].get('httpGet') is not None and
                 healthchecks['livenessProbe']['httpGet'].get('port') is None
             ):
-                healthchecks['livenessProbe']['httpGet']['port'] = env['PORT']
+                healthchecks['livenessProbe']['httpGet']['port'] = env.get(
+                    'PORT', DEFAULT_CONTAINER_PORT)
             container.update(healthchecks)
-        elif kwargs.get('routable', False):
-            container.update(self._default_container_readiness_probe(env.get('PORT', 5000)))
-
-    @staticmethod
-    def _set_lifecycle_hooks(container, env, **kwargs):
-        app_type = kwargs.get("app_type")
-        lifecycle_post_start = kwargs.get('lifecycle_post_start', {})
-        lifecycle_post_start = lifecycle_post_start.get(app_type)
-        lifecycle_pre_stop = kwargs.get('lifecycle_pre_stop', {})
-        lifecycle_pre_stop = lifecycle_pre_stop.get(app_type)
-        if lifecycle_post_start or lifecycle_pre_stop:
-            lifecycle = defaultdict(dict)
-
-            if lifecycle_post_start:
-                lifecycle["postStart"] = {
-                        'exec': {
-                            "command": [
-                                "/bin/bash",
-                                "-c",
-                                "{0}".format(lifecycle_post_start)
-                            ]
-                        }
-                }
-            if lifecycle_pre_stop:
-                lifecycle["preStop"] = {
-                        'exec': {
-                            "command": [
-                                "/bin/bash",
-                                "-c",
-                                "{0}".format(lifecycle_pre_stop)
-                            ]
-                        }
-                }
-            container["lifecycle"] = dict(lifecycle)
+        if kwargs.get('routable', False) and healthchecks.get('readinessProbe') is None:
+            # If routable, set the default probe
+            container.update(
+                self._default_container_readiness_probe(env.get('PORT', DEFAULT_CONTAINER_PORT)))
 
     @staticmethod
     def _default_container_readiness_probe(port, delay=5, timeout=5, period_seconds=5,
@@ -370,7 +294,6 @@ def _default_container_readiness_probe(port, delay=5, timeout=5, period_seconds=
         """
         readinessprobe = {
             'readinessProbe': {
-                # an exec probe
                 'tcpSocket': {
                     "port": int(port)
                 },
@@ -448,7 +371,7 @@ def readiness_status(self, pod):
         """Check if the pod container have passed the readiness probes"""
         name = '{}-{}'.format(pod['metadata']['labels']['app'], pod['metadata']['labels']['type'])
         # find the right container in case there are many on the pod
-        container = self.find_container(name, pod['status']['containerStatuses'])
+        container = self.find_container(name, pod['status'].get('containerStatuses', []))
         if container is None:
             # Seems like the most sensible default
             return 'Unknown'
@@ -489,7 +412,7 @@ def deleted(self, pod):
         if 'deletionTimestamp' in pod['metadata']:
             # past the graceful deletion period
             deletion = self.parse_date(pod['metadata']['deletionTimestamp'])
-            if deletion < datetime.utcnow():
+            if deletion < datetime.now(timezone.utc):
                 return True
 
         return False
@@ -501,7 +424,7 @@ def pending_status(self, pod):
 
         name = '{}-{}'.format(pod['metadata']['labels']['app'], pod['metadata']['labels']['type'])
         # find the right container in case there are many on the pod
-        container = self.pod.find_container(name, pod['status']['containerStatuses'])
+        container = self.find_container(name, pod['status'].get('containerStatuses', []))
         if container is None:
             # Return Pending if nothing else can be found
             return 'Pending', ''
@@ -521,7 +444,7 @@ def pending_status(self, pod):
                     return reason, message
 
                 event = events.pop()
-                return event['reason'], event['message']
+                return event['reason'], event['note']
 
             return reason, message
 
@@ -532,17 +455,16 @@ def events(self, pod):
         """Process events for a given Pod to find if Pulling is happening, among other events"""
         # fetch all events for this pod
         fields = {
-            'involvedObject.name': pod['metadata']['name'],
-            'involvedObject.namespace': pod['metadata']['namespace'],
-            'involvedObject.uid': pod['metadata']['uid']
+            'regarding.name': pod['metadata']['name'],
+            'regarding.namespace': pod['metadata']['namespace'],
+            'regarding.uid': pod['metadata']['uid']
         }
-        events = self.ns.events(
+        events = self.ev.get(
             pod['metadata']['namespace'], fields=fields).json()['items']
         if not events:
             events = []
         # make sure that events are sorted
-        events.sort(
-            key=lambda x: x['lastTimestamp'] if x['lastTimestamp'] else "")
+        events.sort(key=lambda x: x['metadata']['creationTimestamp'] if x['metadata']['creationTimestamp'] else "")  # noqa
         return events
 
     def _handle_pod_errors(self, pod, reason, message):
@@ -559,21 +481,22 @@ def _handle_pod_errors(self, pod, reason, message):
             'ImagePullBackOff',
             'RegistryUnavailable',
             'ErrImageInspect',
+            "CreateContainerError",
+            "CrashLoopBackOff",
         ]
         # Image event reason mapping
         event_errors = {
             "Failed": "FailedToPullImage",
             "InspectFailed": "FailedToInspectImage",
             "ErrImageNeverPull": "ErrImageNeverPullPolicy",
-            # Not including this one for now as the message is not useful
-            # "BackOff": "BackOffPullImage",
+            "BackOff": "CrashLoopBackOff",
             # FailedScheduling relates limits
             "FailedScheduling": "FailedScheduling",
         }
 
         # Nicer error than from the event
         # Often this gets to ImageBullBackOff before we can introspect tho
-        if reason == 'ErrImagePull':
+        if reason in ['ErrImagePull', 'CreateContainerError']:
             raise KubeException(message)
 
         # collect all error messages of worth
@@ -584,7 +507,7 @@ def _handle_pod_errors(self, pod, reason, message):
                     # only show a given error once
                     event_errors.pop(event['reason'])
                     # strip out whitespaces on either side
-                    message = "\n".join([x.strip() for x in event['message'].split("\n")])
+                    message = "\n".join([x.strip() for x in event['note'].split("\n")])
                     messages.append(message)
 
         if messages:
@@ -609,7 +532,7 @@ def _handle_long_image_pulling(self, reason, pod):
         # see if pull operation has been happening for over 1 minute
         seconds = 60  # time threshold before padding timeout
         start = self.parse_date(event['firstTimestamp'])
-        if (start + timedelta(seconds=seconds)) < datetime.utcnow():
+        if (start + timedelta(seconds=seconds)) < datetime.now(timezone.utc):
             # make it so function doesn't do processing again
             setattr(self, '_handle_long_image_pulling_applied', True)
             return 600
@@ -628,7 +551,13 @@ def _handle_pending_pods(self, namespace, labels):
             pods = []
         for pod in pods:
             # only care about pods that are not starting or in the starting phases
-            if pod['status']['phase'] not in ['Pending', 'ContainerCreating']:
+            phase = pod['status']['phase']
+            name = '{}-{}'.format(pod['metadata']['labels']['app'],
+                                  pod['metadata']['labels']['type'])
+            container = self.find_container(name, pod['status'].get('containerStatuses', []))
+            # phase is Running, but state is waiting in CrashLoopBackOff
+            if phase not in ['Pending', 'ContainerCreating'] and \
+                    (phase == 'Running' and 'waiting' not in container['state'].keys()):
                 continue
 
             # Get more information on why a pod is pending
@@ -747,6 +676,21 @@ def wait_until_ready(self, namespace, containers, labels, desired, timeout):  #
 
         self.log(namespace, "{} out of {} pods are in service".format(count, desired))  # noqa
 
+    def watch(self, namespace, labels, timeout=300, interval=5,
+              until_states=['down', 'crashed', 'error']):
+        self.log(
+            namespace,
+            f"watching for {labels} labels in {namespace} namespace, "
+            f"timeout: {timeout}, interval: {interval}, until_states: {until_states}"
+        )
+        for _ in range(math.ceil(timeout / interval)):
+            for p in self.get(namespace, labels=labels).json()['items']:
+                state = str(self.state(p))
+                yield state
+                if state in until_states:
+                    return
+            time.sleep(interval)
+
     def _handle_not_ready_pods(self, namespace, labels):
         """
         Detects if any pod is in the Running phase but not Ready and handles
@@ -762,43 +706,51 @@ def _handle_not_ready_pods(self, namespace, labels):
 
             name = '{}-{}'.format(pod['metadata']['labels']['app'], pod['metadata']['labels']['type'])  # noqa
             # find the right container in case there are many on the pod
-            container = self.find_container(name, pod['status']['containerStatuses'])
+            container = self.find_container(name, pod['status'].get('containerStatuses', []))
             if container is None or container['ready'] == 'true':
                 continue
 
             for event in self.events(pod):
                 if event['reason'] == 'Unhealthy':
                     # strip out whitespaces on either side
-                    message = "\n".join([x.strip() for x in event['message'].split("\n")])
+                    message = "\n".join([x.strip() for x in event['note'].split("\n")])
                     raise KubeException(message)
 
+    def _get_probe_timeout(self, probe):
+        period_seconds = probe.get('periodSeconds', 10)
+        timeout_seconds = probe.get('timeoutSeconds', 1)
+        failure_threshold = probe.get('failureThreshold', 3)
+        success_threshold = probe.get('successThreshold', 1)
+        initial_delay_seconds = probe.get('initialDelaySeconds', 0)
+        if period_seconds > initial_delay_seconds:
+            timeout = 0
+        else:
+            timeout = initial_delay_seconds
+        timeout += (period_seconds + timeout_seconds) * failure_threshold
+        timeout += (period_seconds + timeout_seconds) * success_threshold
+        return int(timeout)
+
     def deploy_probe_timeout(self, timeout, namespace, labels, containers):
         """
         Added in additional timeouts based on readiness and liveness probe
 
         Uses the max of the two instead of combining them as the checks are stacked.
         """
-
         container_name = '{}-{}'.format(labels['app'], labels['type'])
         container = self.pod.find_container(container_name, containers)
 
         # get health info from container
         added_timeout = []
         if 'readinessProbe' in container:
-            # If there is initial delay on the readiness check then timeout needs to be higher
-            # this is to account for kubernetes having readiness check report as failure until
-            # the initial delay period is up
-            added_timeout.append(int(container['readinessProbe'].get('initialDelaySeconds', 50)))
-
+            added_timeout.append(self._get_probe_timeout(container['readinessProbe']))
         if 'livenessProbe' in container:
-            # If there is initial delay on the readiness check then timeout needs to be higher
-            # this is to account for kubernetes having liveness check report as failure until
-            # the initial delay period is up
-            added_timeout.append(int(container['livenessProbe'].get('initialDelaySeconds', 50)))
-
+            added_timeout.append(self._get_probe_timeout(container['livenessProbe']))
         if added_timeout:
             delay = max(added_timeout)
-            self.log(namespace, "adding {}s on to the original {}s timeout to account for the initial delay specified in the liveness / readiness probe".format(delay, timeout))  # noqa
+            self.log(
+                namespace,
+                "adding {}s on to the original {}s timeout for the probe".format(delay, timeout))
             timeout += delay
-
+        if 'startupProbe' in container:
+            timeout += self._get_probe_timeout(container['startupProbe'])
         return timeout
diff --git a/rootfs/scheduler/resources/pv.py b/rootfs/scheduler/resources/pv.py
new file mode 100644
index 000000000..56f6a9e27
--- /dev/null
+++ b/rootfs/scheduler/resources/pv.py
@@ -0,0 +1,75 @@
+import json
+import logging
+from api import utils
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class PersistentVolume(Resource):
+    short_name = 'pv'
+
+    def manifest(self, name, version=None, **kwargs):
+        labels = {
+            'heritage': 'drycc',
+        }
+        data = {
+            "apiVersion": self.api_version,
+            "kind": "PersistentVolume",
+            "metadata": {
+                "name": name,
+                'labels': labels
+            },
+        }
+        data = utils.dict_merge(data, kwargs)
+        if version:
+            data["metadata"]["resourceVersion"] = version
+        return data
+
+    def get(self, name=None):
+        """
+        Fetch a single persistentvolume or a list of persistentvolumes
+        """
+        if name is not None:
+            url = self.api('/persistentvolumes/{}', name)
+            message = 'get persistentvolume ' + name
+        else:
+            url = self.api('/persistentvolumes')
+            message = 'get persistentvolumes'
+        response = self.http_get(url)
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response, message)
+        return response
+
+    def create(self, name, **kwargs):
+        """
+        Create persistentvolume
+        """
+        url = self.api('/persistentvolumes')
+        data = self.manifest(name, **kwargs)
+        response = self.http_post(url, json=data)
+        if not response.status_code == 201:
+            raise KubeHTTPException(
+                response,
+                "create persistentvolume"
+            )
+        return response
+
+    def patch(self, name, **kwargs):
+        url = self.api('/persistentvolumes/{}', name)
+        data = self.manifest(name, **kwargs)
+        response = self.http_patch(url, json=data, headers={"Content-Type": "application/merge-patch+json"})  # noqa
+        if self.unhealthy(response.status_code):
+            self.log('template used: {}'.format(json.dumps(data, indent=4)), logging.DEBUG)
+            raise KubeHTTPException(response, 'update persistentvolume "{}"', name)
+        return response
+
+    def delete(self, name):
+        """
+        Delete persistentvolume
+        """
+        url = self.api('/persistentvolumes/{}', name)
+        response = self.http_delete(url)
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response,
+                                    'delete persistentvolume ' + name)
+        return response
diff --git a/rootfs/scheduler/resources/pvc.py b/rootfs/scheduler/resources/pvc.py
index 4ccfbf629..8582f2016 100644
--- a/rootfs/scheduler/resources/pvc.py
+++ b/rootfs/scheduler/resources/pvc.py
@@ -1,4 +1,6 @@
 import json
+import logging
+from api import utils
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -17,20 +19,9 @@ def manifest(self, namespace, name, version=None, **kwargs):
                 "name": name,
                 "namespace": namespace,
                 'labels': labels
-            },
-            "spec": {
-                "accessModes": [
-                    "ReadWriteMany"
-                ],
-                "resources": {
-                    "requests": {
-                        "storage": kwargs.get('size')
-                    },
-                },
-                "storageClassName": kwargs.get("storage_class"),
-                "volumeMode": "Filesystem",
             }
         }
+        data = utils.dict_merge(data, kwargs)
         if version:
             data["metadata"]["resourceVersion"] = version
         return data
@@ -70,7 +61,7 @@ def patch(self, namespace, name, **kwargs):
         data = self.manifest(namespace, name, **kwargs)
         response = self.http_patch(url, json=data, headers={"Content-Type": "application/merge-patch+json"})  # noqa
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(data, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(data, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(response, 'update persistentvolumeclaims "{}"', name)
         return response
 
diff --git a/rootfs/scheduler/resources/quota.py b/rootfs/scheduler/resources/quota.py
deleted file mode 100644
index 8e9083fc2..000000000
--- a/rootfs/scheduler/resources/quota.py
+++ /dev/null
@@ -1,45 +0,0 @@
-from scheduler.exceptions import KubeHTTPException
-from scheduler.resources import Resource
-
-
-class Quota(Resource):
-    short_name = 'quota'
-
-    def get(self, namespace_name, name):
-        """
-        Fetch a single quota
-        """
-        url = '/namespaces/{}/resourcequotas/{}'.format(namespace_name, name)
-        message = 'get quota {} for namespace {}'.format(name, namespace_name)
-        url = self.api(url)
-        response = self.http_get(url)
-        if self.unhealthy(response.status_code):
-            raise KubeHTTPException(response, message)
-
-        return response
-
-    def create(self, namespace_name, name, **kwargs):
-        """
-        Create resource quota for namespace
-        """
-        url = self.api("/namespaces/{}/resourcequotas".format(namespace_name))
-        manifest = {
-            "kind": "ResourceQuota",
-            "apiVersion": self.api_version,
-            "metadata": {
-                "namespace": namespace_name,
-                "name": name,
-                'labels': {
-                    'app': namespace_name,
-                    'heritage': 'drycc'
-                },
-            },
-            'spec': kwargs.get('spec', {})
-        }
-        response = self.http_post(url, json=manifest)
-        if not response.status_code == 201:
-            raise KubeHTTPException(response,
-                                    "create quota {} for namespace {}".format(
-                                        name, namespace_name))
-
-        return response
diff --git a/rootfs/scheduler/resources/rc.py b/rootfs/scheduler/resources/rc.py
index 950c5ac1d..f5d1fce56 100644
--- a/rootfs/scheduler/resources/rc.py
+++ b/rootfs/scheduler/resources/rc.py
@@ -1,5 +1,6 @@
 import json
 import time
+import logging
 from scheduler.exceptions import KubeHTTPException
 from scheduler.resources import Resource
 
@@ -28,7 +29,7 @@ def get(self, namespace, name=None, **kwargs):
 
         return response
 
-    def create(self, namespace, name, image, entrypoint, command, **kwargs):
+    def create(self, namespace, name, image, command, args, **kwargs):
         manifest = {
             'kind': 'ReplicationController',
             'apiVersion': self.api_version,
@@ -47,8 +48,8 @@ def create(self, namespace, name, image, entrypoint, command, **kwargs):
         }
 
         # tell pod how to execute the process
-        kwargs['command'] = entrypoint
-        kwargs['args'] = command
+        kwargs['command'] = command
+        kwargs['args'] = args
 
         # pod manifest spec
         manifest['spec']['template'] = self.pod.manifest(namespace, name, image, **kwargs)
@@ -56,7 +57,8 @@ def create(self, namespace, name, image, entrypoint, command, **kwargs):
         url = self.api("/namespaces/{}/replicationcontrollers", namespace)
         resp = self.http_post(url, json=manifest)
         if self.unhealthy(resp.status_code):
-            self.log(namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
             raise KubeHTTPException(
                 resp,
                 'create ReplicationController "{}" in Namespace "{}"', name, namespace
@@ -114,7 +116,7 @@ def wait_until_updated(self, namespace, name):
         More information is also available at:
         https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
-        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
         for _ in range(30):
             try:
                 rc = self.get(namespace, name).json()
@@ -122,7 +124,7 @@ def wait_until_updated(self, namespace, name):
                     "observedGeneration" in rc["status"] and
                     rc["status"]["observedGeneration"] >= rc["metadata"]["generation"]
                 ):
-                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
                     break
 
                 time.sleep(1)
diff --git a/rootfs/scheduler/resources/rs.py b/rootfs/scheduler/resources/rs.py
index bde61d188..289c26949 100644
--- a/rootfs/scheduler/resources/rs.py
+++ b/rootfs/scheduler/resources/rs.py
@@ -1,4 +1,5 @@
 import time
+import logging
 from scheduler.exceptions import KubeHTTPException
 from scheduler.resources import Resource
 
@@ -69,7 +70,7 @@ def wait_until_updated(self, namespace, name):
         More information is also available at:
         https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#metadata
         """
-        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+        self.log(namespace, "waiting for ReplicationController {} to get a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
         for _ in range(30):
             try:
                 rs = self.get(namespace, name).json()
@@ -77,7 +78,7 @@ def wait_until_updated(self, namespace, name):
                     "observedGeneration" in rs["status"] and
                     rs["status"]["observedGeneration"] >= rs["metadata"]["generation"]
                 ):
-                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), 'DEBUG')  # noqa
+                    self.log(namespace, "ReplicationController {} got a newer generation (30s timeout)".format(name), logging.DEBUG)  # noqa
                     break
 
                 time.sleep(1)
diff --git a/rootfs/scheduler/resources/scale.py b/rootfs/scheduler/resources/scale.py
index 43f72eb4c..fd5babcec 100644
--- a/rootfs/scheduler/resources/scale.py
+++ b/rootfs/scheduler/resources/scale.py
@@ -1,4 +1,5 @@
 import json
+import logging
 from scheduler.resources import Resource
 from scheduler.exceptions import KubeHTTPException
 
@@ -32,7 +33,7 @@ def update(self, namespace, name, replicas, target):
         url = self.api("/namespaces/{}/{}/{}/scale", namespace, resource_type, name)
         response = self.http_put(url, json=manifest)
         if self.unhealthy(response.status_code):
-            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), 'DEBUG')  # noqa
+            self.log(namespace, 'template used: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)  # noqa
             raise KubeHTTPException(
                 response,
                 'scale {} "{}" in Namespace "{}"', target['kind'], name, namespace
diff --git a/rootfs/scheduler/resources/secret.py b/rootfs/scheduler/resources/secret.py
index ad7d3f1eb..41424322c 100644
--- a/rootfs/scheduler/resources/secret.py
+++ b/rootfs/scheduler/resources/secret.py
@@ -117,10 +117,10 @@ def update(self, namespace, name, data, secret_type='Opaque', labels={}):
 
         return response
 
-    def delete(self, namespace, name):
+    def delete(self, namespace, name, ignore_exception=False):
         url = self.api("/namespaces/{}/secrets/{}", namespace, name)
         response = self.http_delete(url)
-        if self.unhealthy(response.status_code):
+        if not ignore_exception and self.unhealthy(response.status_code):
             raise KubeHTTPException(
                 response,
                 'delete Secret "{}" in Namespace "{}"', name, namespace
diff --git a/rootfs/scheduler/resources/service.py b/rootfs/scheduler/resources/service.py
index 28f22e360..f8d5e1745 100644
--- a/rootfs/scheduler/resources/service.py
+++ b/rootfs/scheduler/resources/service.py
@@ -28,8 +28,8 @@ def manifest(self, namespace, name, **kwargs):
         }
         if "version" in kwargs:
             data["metadata"]["resourceVersion"] = kwargs.get("version")
-        if "procfile_type" in kwargs:
-            data["spec"]["selector"]['type'] = kwargs.get("procfile_type")
+        if "ptype" in kwargs:
+            data["spec"]["selector"]['type'] = kwargs.get("ptype")
         return data
 
     def get(self, namespace, name=None, **kwargs):
@@ -88,10 +88,10 @@ def update(self, namespace, name, data):
 
         return response
 
-    def delete(self, namespace, name):
+    def delete(self, namespace, name, ignore_exception=False):
         url = self.api("/namespaces/{}/services/{}", namespace, name)
         response = self.http_delete(url)
-        if self.unhealthy(response.status_code):
+        if not ignore_exception and self.unhealthy(response.status_code):
             raise KubeHTTPException(
                 response,
                 'delete Service "{}" in Namespace "{}"', name, namespace
diff --git a/rootfs/scheduler/resources/statefulset.py b/rootfs/scheduler/resources/statefulset.py
new file mode 100644
index 000000000..e7b0cff8d
--- /dev/null
+++ b/rootfs/scheduler/resources/statefulset.py
@@ -0,0 +1,45 @@
+import json
+import logging
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException
+
+
+class Statefulset(Resource):
+    api_prefix = 'apis'
+    api_version = 'apps/v1'
+
+    def get(self, namespace, name=None, ignore_exception=False, **kwargs):
+        """
+        Fetch a single Statefulsets or a list
+        """
+        url = '/namespaces/{}/statefulsets'
+        args = [namespace]
+        if name is not None:
+            args.append(name)
+            url += '/{}'
+            message = 'get Statefulset "{}" in Namespace "{}"'
+        else:
+            message = 'get Statefulsets in Namespace "{}"'
+
+        url = self.api(url, *args)
+        response = self.http_get(url, params=self.query_params(**kwargs))
+        if self.unhealthy(response.status_code) and not ignore_exception:
+            args.reverse()  # error msg is in reverse order
+            raise KubeHTTPException(response, message, *args)
+
+        return response
+
+    def patch(self, namespace, name, manifest, ignore_exception=False, **kwargs):
+        url = self.api("/namespaces/{}/statefulsets/{}", namespace, name)
+        response = self.http_patch(
+            url,
+            json=manifest,
+            headers={"Content-Type": "application/merge-patch+json"}
+        )
+
+        if self.unhealthy(response.status_code):
+            self.log(
+                namespace, 'template: {}'.format(json.dumps(manifest, indent=4)), logging.DEBUG)
+            if not ignore_exception:
+                raise KubeHTTPException(response, 'patch Statefulset "{}"', name)
+        return response
diff --git a/rootfs/scheduler/tests/__init__.py b/rootfs/scheduler/tests/__init__.py
index 11db26bd9..831c7c0f9 100644
--- a/rootfs/scheduler/tests/__init__.py
+++ b/rootfs/scheduler/tests/__init__.py
@@ -3,7 +3,7 @@
 from django.conf import settings
 
 from scheduler import mock
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class TestCase(DjangoTestCase):
@@ -24,14 +24,12 @@ def create_namespace(self):
         data = response.json()
         self.assertEqual(data['apiVersion'], 'v1')
         self.assertEqual(data['kind'], 'Namespace')
-        self.assertDictContainsSubset(
-            {
-                'name': namespace,
-                'labels': {
-                    'heritage': 'drycc'
-                }
-            },
-            data['metadata']
-        )
+        metadata = {
+            'name': namespace,
+            'labels': {
+                'heritage': 'drycc'
+            }
+        }
+        self.assertEqual(data['metadata'], metadata | data['metadata'])
 
         return namespace
diff --git a/rootfs/scheduler/tests/test_daemonsets.py b/rootfs/scheduler/tests/test_daemonsets.py
new file mode 100644
index 000000000..f4c8ab985
--- /dev/null
+++ b/rootfs/scheduler/tests/test_daemonsets.py
@@ -0,0 +1,72 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with "./manage.py test scheduler"
+"""
+from scheduler.tests import TestCase
+from scheduler import KubeHTTPException
+
+
+class DaemonsetsTest(TestCase):
+    """Tests scheduler daemonset calls"""
+
+    def test_get_daemonsets(self):
+        response = self.scheduler.daemonset.get('drycc')
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertIn('items', data)
+        self.assertEqual(1, len(data['items']))
+        # simple verify of data
+        self.assertEqual(data['items'][0]['metadata']['name'], 'drycc')
+
+    def test_get_daemonset(self):
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to get Daemonset doesnotexist in Namespace drycc: 404 Not Found'  # noqa
+        ):
+            self.scheduler.daemonset.get('drycc', 'doesnotexist')
+        namespace = 'drycc'
+        name = 'drycc'
+        response = self.scheduler.daemonset.get(namespace, name)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertEqual(data['apiVersion'], 'apps/v1')
+        self.assertEqual(data['kind'], 'DaemonSet')
+        self.assertEqual(data['metadata']['name'], name)
+
+    def test_patch_daemonset(self):
+        namespace = 'drycc'
+        name = 'drycc'
+        manifest_affinity = {
+            "nodeAffinity": {
+                "requiredDuringSchedulingIgnoredDuringExecution": {
+                    "nodeSelectorTerms": [{
+                        "matchExpressions": [{
+                            "key": "kubernetes.io/hostname",
+                            "operator": "In",
+                            "values": [
+                                "nohostname"
+                            ]
+                        }]
+                    }]
+                }
+            }
+        }
+        response = self.scheduler.daemonset.patch(
+            namespace,
+            name,
+            manifest={
+                "spec": {
+                    "template": {
+                        "spec": {
+                            "affinity": manifest_affinity
+                        }
+                    }
+                }
+            },
+        )
+        affinity = response.json()["spec"]["template"]['spec']['affinity']
+        self.assertEqual(
+            affinity,
+            manifest_affinity
+        )
diff --git a/rootfs/scheduler/tests/test_deployments.py b/rootfs/scheduler/tests/test_deployments.py
index e40c66b6f..636e1ff64 100644
--- a/rootfs/scheduler/tests/test_deployments.py
+++ b/rootfs/scheduler/tests/test_deployments.py
@@ -8,7 +8,7 @@
 from packaging.version import parse, Version, InvalidVersion
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class DeploymentsTest(TestCase):
@@ -24,10 +24,10 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'spec_annotations': kwargs.get('spec_annotations', {}),
         }
 
@@ -45,10 +45,10 @@ def update(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'spec_annotations': kwargs.get('spec_annotations', {}),
         }
 
@@ -67,10 +67,10 @@ def scale(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
         }
 
         self.scheduler.scale(namespace, name, **kwargs)
@@ -94,7 +94,7 @@ def test_bad_init_api_version(self):
     def test_deployment_api_version_1_9_and_up(self):
         cases = ['1.12', '1.11', '1.10', '1.9']
 
-        deployment = copy.copy(self.scheduler.deployment)
+        deployment = copy.deepcopy(self.scheduler.deployment)
 
         expected = 'apps/v1'
 
@@ -112,7 +112,7 @@ def test_deployment_api_version_1_9_and_up(self):
     def test_deployment_api_version_1_8_and_lower(self):
         cases = ['1.8', '1.7', '1.6', '1.5', '1.4', '1.3', '1.2']
 
-        deployment = copy.copy(self.scheduler.deployment)
+        deployment = copy.deepcopy(self.scheduler.deployment)
 
         expected = 'apps/v1'
 
@@ -185,7 +185,7 @@ def test_get_deployments(self):
         data = response.json()
         self.assertEqual(response.status_code, 200, data)
         self.assertIn('items', data)
-        self.assertEqual(data['items'], None)
+        self.assertEqual(data['items'], [])
         # test success
         name = self.create()
         response = self.scheduler.deployment.get(self.namespace)
@@ -213,13 +213,11 @@ def test_get_deployment(self):
         self.assertEqual(data['apiVersion'], 'apps/v1')
         self.assertEqual(data['kind'], 'Deployment')
         self.assertEqual(data['metadata']['name'], name)
-        self.assertDictContainsSubset(
-            {
-                'app': self.namespace,
-                'heritage': 'drycc'
-            },
-            data['metadata']['labels']
-        )
+        labels = {
+            'app': self.namespace,
+            'heritage': 'drycc'
+        }
+        self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels)
 
     def test_scale(self):
         name = self.scale()
@@ -288,14 +286,11 @@ def test_get_deployment_replicaset(self):
         self.assertEqual(data['apiVersion'], 'apps/v1', data)
         self.assertEqual(data['kind'], 'ReplicaSet', data)
         self.assertEqual(data['metadata']['name'], replica_name, data)
-        self.assertDictContainsSubset(
-            {
-                'app': self.namespace,
-                'heritage': 'drycc'
-            },
-            data['metadata']['labels'],
-            data
-        )
+        labels = {
+            'app': self.namespace,
+            'heritage': 'drycc'
+        }
+        self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels, data)
 
     def test_get_deployment_annotations(self):
         """
@@ -307,12 +302,12 @@ def test_get_deployment_annotations(self):
         }
         deployment = self.create(**kwargs)
         data = self.scheduler.deployment.get(self.namespace, deployment).json()
-
-        self.assertDictContainsSubset(
-            {
-                'iam.amazonaws.com/role': 'role-arn'
-            },
-            data['spec']['template']['metadata']['annotations']
+        annotations = {
+            'iam.amazonaws.com/role': 'role-arn'
+        }
+        self.assertEqual(
+            data['spec']['template']['metadata']['annotations'],
+            data['spec']['template']['metadata']['annotations'] | annotations,
         )
 
     def test_get_pod_annotations(self):
@@ -347,20 +342,20 @@ def test_check_for_failed_events(self):
         deployment = self.scheduler.deployment.get(self.namespace, deploy_name).json()
         response = self.scheduler.rs.get(self.namespace, labels=deployment['metadata']['labels'])
         rs = response.json()
-        involved_object = {
-            'involvedObject.kind': 'ReplicaSet',
-            'involvedObject.name': rs['items'][0]['metadata']['name'],
-            'involvedObject.namespace': self.namespace,
-            'involvedObject.uid': rs['items'][0]['metadata']['uid'],
+        regarding = {
+            'regarding.kind': 'ReplicaSet',
+            'regarding.name': rs['items'][0]['metadata']['name'],
+            'regarding.namespace': self.namespace,
+            'regarding.uid': rs['items'][0]['metadata']['uid'],
         }
         message = 'Quota exeeded'
         self.scheduler.ev.create(self.namespace,
                                  '{}'.format(generate_random_name()),
                                  message,
                                  type='Warning',
-                                 involved_object=involved_object,
+                                 regarding=regarding,
                                  reason='FailedCreate')
         with self.assertRaisesRegex(KubeException,
-                                    'Message:{}.*'.format(message)):
+                                    'Message: {}.*'.format(message)):
             self.scheduler.deployment._check_for_failed_events(self.namespace,
                                                                labels=deployment['metadata']['labels'])  # noqa
diff --git a/rootfs/scheduler/tests/test_events.py b/rootfs/scheduler/tests/test_events.py
new file mode 100644
index 000000000..89b17e276
--- /dev/null
+++ b/rootfs/scheduler/tests/test_events.py
@@ -0,0 +1,38 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with './manage.py test scheduler'
+"""
+from scheduler.tests import TestCase
+
+
+class EventTest(TestCase):
+    """Tests scheduler pod calls"""
+
+    def create_event(self, namespace, name, **kwargs):
+        """
+        Helper function to create and verify a events on the namespace
+        """
+        self.scheduler.ns.create(namespace)
+
+        message = "Scaled down replica set test-869947c55f to 1 from 2"
+        # these are all required even if it is kwargs...
+        kwargs = {
+            'reason': 'ScalingReplicaSet',
+            'type': 'Normal'
+        }
+        return self.scheduler.events.create(
+            namespace,
+            name,
+            message,
+            **kwargs
+        )
+
+    def test_create(self):
+        response = self.create_event("test-event", "test-event")
+        self.assertEqual(response.status_code, 201)
+
+    def test_get_events(self):
+        # test success
+        response = self.scheduler.events.get("test-event", "test-event")
+        self.assertEqual(response.status_code, 200)
diff --git a/rootfs/scheduler/tests/test_gateway.py b/rootfs/scheduler/tests/test_gateway.py
index 72d00c2cf..385abb2fa 100644
--- a/rootfs/scheduler/tests/test_gateway.py
+++ b/rootfs/scheduler/tests/test_gateway.py
@@ -75,7 +75,7 @@ def create_http_route(self, name="test-gateway"):
             name,
             name,
             port=5000,
-            procfile_type="web",
+            ptype="web",
             rules={},
             parent_refs={},
         )
@@ -89,11 +89,6 @@ def test_patch_http_route(self):
                         "name": "tcp-echo-v1",
                         "port": 9000,
                         "weight": 10,
-                    },
-                    {
-                        "name": "tcp-echo-v1-canary",
-                        "port": 9000,
-                        "weight": 90,
                     }
                 ]
             }
@@ -102,7 +97,7 @@ def test_patch_http_route(self):
             "test-gateway",
             "test-gateway",
             port=8000,
-            procfile_type="cmd",
+            ptype="cmd",
             version=1,
             rules=rules,
             parent_refs={},
@@ -129,7 +124,7 @@ def create_tcp_route(self, name="test-tcp-route"):
             name,
             name,
             port=5000,
-            procfile_type="celery",
+            ptype="celery",
             rules={},
             parent_refs={},
         )
@@ -143,11 +138,6 @@ def test_patch_tcp_route(self):
                         "name": "tcp-echo-v1",
                         "port": 9000,
                         "weight": 10,
-                    },
-                    {
-                        "name": "tcp-echo-v1-canary",
-                        "port": 9000,
-                        "weight": 90,
                     }
                 ]
             }
@@ -156,7 +146,7 @@ def test_patch_tcp_route(self):
             "test-tcp-route",
             "test-tcp-route",
             port=8000,
-            procfile_type="cmd",
+            ptype="cmd",
             version=1,
             rules=rules,
             parent_refs={},
@@ -183,7 +173,7 @@ def create_udp_route(self, name="test-udp-route"):
             name,
             name,
             port=5000,
-            procfile_type="celery",
+            ptype="celery",
             rules={},
             parent_refs={}
         )
@@ -197,11 +187,6 @@ def test_patch_udp_route(self):
                         "name": "udp-echo-v1",
                         "port": 9000,
                         "weight": 10,
-                    },
-                    {
-                        "name": "udp-echo-v1-canary",
-                        "port": 9000,
-                        "weight": 90,
                     }
                 ]
             }
@@ -210,7 +195,7 @@ def test_patch_udp_route(self):
             "test-udp-route",
             "test-udp-route",
             port=8000,
-            procfile_type="cmd",
+            ptype="cmd",
             version=1,
             rules=rules,
             parent_refs={}
diff --git a/rootfs/scheduler/tests/test_hpa.py b/rootfs/scheduler/tests/test_hpa.py
index 665f3900e..832d98e46 100644
--- a/rootfs/scheduler/tests/test_hpa.py
+++ b/rootfs/scheduler/tests/test_hpa.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class HorizontalPodAutoscalersTest(TestCase):
@@ -23,10 +23,10 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 1),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'spec_annotations': kwargs.get('spec_annotations', {}),
         }
 
@@ -72,10 +72,10 @@ def update_deployment(self, namespace=None, name=generate_random_name(), **kwarg
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'spec_annotations': kwargs.get('spec_annotations', {}),
         }
 
@@ -211,10 +211,8 @@ def test_get_horizontalpodautoscaler(self):
         self.assertEqual(data['apiVersion'], 'autoscaling/v1')
         self.assertEqual(data['kind'], 'HorizontalPodAutoscaler')
         self.assertEqual(data['metadata']['name'], name)
-        self.assertDictContainsSubset(
-            {
-                'app': self.namespace,
-                'heritage': 'drycc'
-            },
-            data['metadata']['labels']
-        )
+        labels = {
+            'app': self.namespace,
+            'heritage': 'drycc'
+        }
+        self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels)
diff --git a/rootfs/scheduler/tests/test_hpa_12_lower.py b/rootfs/scheduler/tests/test_hpa_12_lower.py
index 7d98ba5f7..5cbca133e 100644
--- a/rootfs/scheduler/tests/test_hpa_12_lower.py
+++ b/rootfs/scheduler/tests/test_hpa_12_lower.py
@@ -8,7 +8,7 @@
 
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 @mock.patch('scheduler.KubeHTTPClient.version', lambda *args: Version('1.2'))
@@ -27,10 +27,10 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 1),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'spec_annotations': kwargs.get('spec_annotations', {}),
         }
 
@@ -76,10 +76,10 @@ def update_deployment(self, namespace=None, name=generate_random_name(), **kwarg
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'spec_annotations': kwargs.get('spec_annotations', {}),
         }
 
@@ -215,10 +215,8 @@ def test_get_horizontalpodautoscaler(self):
         self.assertEqual(data['apiVersion'], 'autoscaling/v1')
         self.assertEqual(data['kind'], 'HorizontalPodAutoscaler')
         self.assertEqual(data['metadata']['name'], name)
-        self.assertDictContainsSubset(
-            {
-                'app': self.namespace,
-                'heritage': 'drycc'
-            },
-            data['metadata']['labels']
-        )
+        labels = {
+            'app': self.namespace,
+            'heritage': 'drycc'
+        }
+        self.assertEqual(data['metadata']['labels'], labels | data['metadata']['labels'])
diff --git a/rootfs/scheduler/tests/test_job.py b/rootfs/scheduler/tests/test_job.py
new file mode 100644
index 000000000..b1a25f5cb
--- /dev/null
+++ b/rootfs/scheduler/tests/test_job.py
@@ -0,0 +1,33 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with './manage.py test scheduler'
+"""
+from scheduler.tests import TestCase
+from scheduler.tests.utils import generate_random_name
+
+
+class JobTest(TestCase):
+    """Tests scheduler pod calls"""
+
+    def create(self, namespace=None, name=generate_random_name(), **kwargs):
+        """
+        Helper function to create and verify a pod on the namespace
+        """
+        namespace = self.namespace if namespace is None else namespace
+        # these are all required even if it is kwargs...
+        kwargs = {
+            'app_type': kwargs.get('app_type', 'web'),
+            'version': kwargs.get('version', 'v99'),
+            'image': 'quay.io/fake/image',
+            'command': 'sh',
+            'args': 'start',
+            'deploy_timeout': 10,
+        }
+
+        job = self.scheduler.job.create(namespace, name, **kwargs)
+        self.assertEqual(job.status_code, 201, job.json())
+        return name
+
+    def test_create(self):
+        self.create()
diff --git a/rootfs/scheduler/tests/test_limits.py b/rootfs/scheduler/tests/test_limits.py
deleted file mode 100644
index dc1f99f03..000000000
--- a/rootfs/scheduler/tests/test_limits.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-Unit tests for the Drycc scheduler module.
-
-Run the tests with './manage.py test scheduler'
-"""
-from scheduler.tests import TestCase
-from scheduler import KubeHTTPException
-
-
-class LimitRangesTest(TestCase):
-
-    def test_create_quota(self):
-        namespace_name = self.create_namespace()
-        spec = {
-            "limits": [
-                {
-                    "type": "Container",
-                    "max": {
-                        "cpu": "32",
-                        "memory": "128Gi"
-                    },
-                    "min": {
-                        "cpu": "100m",
-                        "memory": "128Mi"
-                    }
-                },
-                {
-                    "type": "PersistentVolumeClaim",
-                    "max": {
-                        "storage": "100Gi"
-                    },
-                    "min": {
-                        "storage": "100Mi"
-                    }
-                }
-            ]
-        }
-        self.scheduler.limits.create(namespace_name, 'test-1', spec=spec)
-        response = self.scheduler.limits.get(namespace_name, 'test-1')
-        data = response.json()
-        self.assertEqual(data.get('spec', {}), spec)
-        self.assertEqual(data['metadata']['namespace'], namespace_name)
-
-    def test_create_with_nonexistent_namespace(self):
-        with self.assertRaises(
-            KubeHTTPException,
-            msg='failed to create LimitRanges test-1 for namespace ghost-namespace: 404 Not Found'
-        ):
-            self.scheduler.quota.create('ghost-namespace', 'test-1', data={})
diff --git a/rootfs/scheduler/tests/test_namespaces.py b/rootfs/scheduler/tests/test_namespaces.py
index 33f17adc4..7a2c098aa 100644
--- a/rootfs/scheduler/tests/test_namespaces.py
+++ b/rootfs/scheduler/tests/test_namespaces.py
@@ -36,15 +36,13 @@ def test_get_namespace(self):
         self.assertEqual(response.status_code, 200, data)
         self.assertEqual(data['apiVersion'], 'v1')
         self.assertEqual(data['kind'], 'Namespace')
-        self.assertDictContainsSubset(
-            {
-                'name': self.namespace,
-                'labels': {
-                    'heritage': 'drycc'
-                }
-            },
-            data['metadata']
-        )
+        metadata = {
+            'name': self.namespace,
+            'labels': {
+                'heritage': 'drycc'
+            }
+        }
+        self.assertEqual(data['metadata'], data['metadata'] | metadata)
 
     def test_delete_failure(self):
         # test failure
diff --git a/rootfs/scheduler/tests/test_nodes.py b/rootfs/scheduler/tests/test_nodes.py
index 6dd2bd018..adcdec1a1 100644
--- a/rootfs/scheduler/tests/test_nodes.py
+++ b/rootfs/scheduler/tests/test_nodes.py
@@ -34,4 +34,5 @@ def test_get_node(self):
         self.assertEqual(data['apiVersion'], 'v1')
         self.assertEqual(data['kind'], 'Node')
         self.assertEqual(data['metadata']['name'], name)
-        self.assertDictContainsSubset({'ssd': 'true'}, data['metadata']['labels'])
+        labels = {'ssd': 'true'}
+        self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels)
diff --git a/rootfs/scheduler/tests/test_pod_resources.py b/rootfs/scheduler/tests/test_pod_resources.py
index 7c0f0e885..5dade6c37 100644
--- a/rootfs/scheduler/tests/test_pod_resources.py
+++ b/rootfs/scheduler/tests/test_pod_resources.py
@@ -4,46 +4,17 @@
 
 class TestSchedulerPodResources(unittest.TestCase):
     def test_manifest_limits(self):
-        cpu_cases = [
-            {"app_type": "web", "cpu": {"cmd": "2"},
-             "expected": None},
-            {"app_type": "web", "cpu": {"web": "2"},
-             "expected": {"limits": {"cpu": "2"}}},
-            {"app_type": "web", "cpu": {"web": "0/3"},
-             "expected": {"requests": {"cpu": "0"}, "limits": {"cpu": "3"}}},
-            {"app_type": "web", "cpu": {"web": "4/5"},
-             "expected": {"requests": {"cpu": "4"}, "limits": {"cpu": "5"}}},
-            {"app_type": "web", "cpu": {"web": "400m/500m"},
-             "expected": {"requests": {"cpu": "400m"}, "limits": {"cpu": "500m"}}},
-            {"app_type": "web", "cpu": {"web": "0.6/0.7"},
-             "expected": {"requests": {"cpu": "0.6"}, "limits": {"cpu": "0.7"}}},
+        resources_cases = [
+            {"app_type": "web", "expected": {"limits": {"cpu": 1}}},
+            {"app_type": "web", "expected": {"limits": {"memory": "1G"}}},
+            {"app_type": "web", "expected": {"limits": {"cpu": 1, "memory": "1G"}}},
         ]
-
-        mem_cases = [
-            {"app_type": "web", "memory": {"cmd": "2G"},
-             "expected": None},
-            {"app_type": "web", "memory": {"web": "200M"},
-             "expected": {"limits": {"memory": "200Mi"}}},
-            {"app_type": "web", "memory": {"web": "0/3G"},
-             "expected": {"requests": {"memory": "0"}, "limits": {"memory": "3Gi"}}},
-            {"app_type": "web", "memory": {"web": "400M/500MB"},
-             "expected": {"requests": {"memory": "400Mi"}, "limits": {"memory": "500Mi"}}},
-        ]
-
-        for caze in cpu_cases:
-            manifest = Pod("").manifest("",
-                                        "",
-                                        "",
-                                        app_type=caze["app_type"],
-                                        cpu=caze["cpu"])
-            self._assert_resources(caze, manifest)
-
-        for caze in mem_cases:
+        for caze in resources_cases:
             manifest = Pod("").manifest("",
                                         "",
                                         "",
                                         app_type=caze["app_type"],
-                                        memory=caze["memory"])
+                                        resources=caze["expected"])
             self._assert_resources(caze, manifest)
 
     def _assert_resources(self, caze, manifest):
diff --git a/rootfs/scheduler/tests/test_pods.py b/rootfs/scheduler/tests/test_pods.py
index 864b6780d..b1373d6dd 100644
--- a/rootfs/scheduler/tests/test_pods.py
+++ b/rootfs/scheduler/tests/test_pods.py
@@ -3,11 +3,13 @@
 
 Run the tests with './manage.py test scheduler'
 """
+import time
 from unittest import mock
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
+from scheduler.states import PodState
 
 
 class PodsTest(TestCase):
@@ -23,10 +25,10 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
             'deploy_timeout': 10,
         }
 
@@ -86,13 +88,11 @@ def test_get_pod(self):
         self.assertEqual(response.status_code, 200, data)
         self.assertEqual(data['kind'], 'Pod')
         self.assertEqual(data['metadata']['name'], name)
-        self.assertDictContainsSubset(
-            {
-                'app': self.namespace,
-                'heritage': 'drycc'
-            },
-            data['metadata']['labels']
-        )
+        labels = {
+            'app': self.namespace,
+            'heritage': 'drycc'
+        }
+        self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels)
 
     def test_liveness_status(self):
         # Missing Ready type means pod has passed liveness check
@@ -187,6 +187,28 @@ def test_ready(self):
                 liveness.return_value = True
                 self.assertTrue(self.scheduler.pod.ready(pod))
 
+    def test_watch(self):
+        # create a pod to then manipulate
+        name = self.create()
+        pod = self.scheduler.pod.get(self.namespace, name).json()
+        labels = pod['metadata']['labels']
+        with mock.patch('scheduler.resources.pod.Pod.state') as mock_pod:
+            mock_pod.return_value = PodState.creating
+            for count, state in enumerate(self.scheduler.pod.watch(
+                    self.namespace, labels, interval=1)):
+                if count < 3:
+                    self.assertEqual(state, str(PodState.creating))
+                else:
+                    mock_pod.return_value = PodState.down
+            self.assertEqual(state, str(PodState.down))
+            mock_pod.return_value = PodState.creating
+            state = None
+            start_time = time.time()
+            for state in self.scheduler.pod.watch(self.namespace, labels, timeout=3, interval=1):
+                self.assertEqual(state, str(PodState.creating))
+            self.assertEqual(state, str(PodState.creating))
+            self.assertTrue(4 > (time.time() - start_time) > 3)
+
     def test_deleted(self):
         # create a pod to then manipulate
         name = self.create()
@@ -196,6 +218,6 @@ def test_deleted(self):
         self.assertFalse(self.scheduler.pod.deleted(pod))
 
         # set deleted 10 minutes in the past
-        ts_deleted = datetime.utcnow() - timedelta(minutes=10)
+        ts_deleted = datetime.now(timezone.utc) - timedelta(minutes=10)
         pod['metadata']['deletionTimestamp'] = ts_deleted.strftime(self.scheduler.DATETIME_FORMAT)
         self.assertTrue(self.scheduler.pod.deleted(pod))
diff --git a/rootfs/scheduler/tests/test_pvc.py b/rootfs/scheduler/tests/test_pvc.py
index 64a0ce81d..438b90c15 100644
--- a/rootfs/scheduler/tests/test_pvc.py
+++ b/rootfs/scheduler/tests/test_pvc.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class PVCTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_quota.py b/rootfs/scheduler/tests/test_quota.py
deleted file mode 100644
index c3c60f11a..000000000
--- a/rootfs/scheduler/tests/test_quota.py
+++ /dev/null
@@ -1,33 +0,0 @@
-"""
-Unit tests for the Drycc scheduler module.
-
-Run the tests with './manage.py test scheduler'
-"""
-from scheduler.tests import TestCase
-from scheduler import KubeHTTPException
-
-
-class QuotaTest(TestCase):
-
-    def test_create_quota(self):
-        namespace_name = self.create_namespace()
-        spec = {
-            'hard': {
-                'cpu': '3',
-                'pods': '10',
-                'secrets': '5'
-            }
-        }
-        self.scheduler.quota.create(namespace_name, 'test1', spec=spec)
-
-        response = self.scheduler.quota.get(namespace_name, 'test1')
-        data = response.json()
-        self.assertEqual(data.get('spec', {}), spec)
-        self.assertEqual(data['metadata']['namespace'], namespace_name)
-
-    def test_create_with_nonexistent_namespace(self):
-        with self.assertRaises(
-            KubeHTTPException,
-            msg='failed to create quota test1 for namespace ghost-namespace: 404 Not Found'
-        ):
-            self.scheduler.quota.create('ghost-namespace', 'test1', data={})
diff --git a/rootfs/scheduler/tests/test_rc.py b/rootfs/scheduler/tests/test_rc.py
index 97f92adb9..871f903b5 100644
--- a/rootfs/scheduler/tests/test_rc.py
+++ b/rootfs/scheduler/tests/test_rc.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class ReplicationControllersTest(TestCase):
@@ -21,10 +21,10 @@ def create(self, namespace=None, name=generate_random_name(), **kwargs):
             'app_type': kwargs.get('app_type', 'web'),
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
         }
 
         rc = self.scheduler.rc.create(namespace, name, **kwargs)
@@ -43,10 +43,10 @@ def scale_rc(self, namespace=None, name=generate_random_name(), **kwargs):
             'version': kwargs.get('version', 'v99'),
             'replicas': kwargs.get('replicas', 4),
             'deploy_timeout': 120,
-            'pod_termination_grace_period_seconds': 2,
+            'termination_grace_period_seconds': 2,
             'image': 'quay.io/fake/image',
-            'entrypoint': 'sh',
-            'command': 'start',
+            'command': 'sh',
+            'args': 'start',
         }
 
         self.scheduler.scale_rc(namespace, name, **kwargs)
@@ -126,10 +126,8 @@ def test_get_rc(self):
         self.assertEqual(data['apiVersion'], 'v1')
         self.assertEqual(data['kind'], 'ReplicationController')
         self.assertEqual(data['metadata']['name'], name)
-        self.assertDictContainsSubset(
-            {
-                'app': self.namespace,
-                'heritage': 'drycc'
-            },
-            data['metadata']['labels']
-        )
+        labels = {
+            'app': self.namespace,
+            'heritage': 'drycc'
+        }
+        self.assertEqual(data['metadata']['labels'], data['metadata']['labels'] | labels)
diff --git a/rootfs/scheduler/tests/test_scheduler.py b/rootfs/scheduler/tests/test_scheduler.py
index a1627d556..fcfb1176f 100644
--- a/rootfs/scheduler/tests/test_scheduler.py
+++ b/rootfs/scheduler/tests/test_scheduler.py
@@ -35,7 +35,7 @@ def test_set_container_applies_healthcheck_with_routable(self):
         self.scheduler.pod._set_container(
             'foo', 'bar', data, routable=True, healthcheck=healthcheck
         )
-        self.assertDictContainsSubset(healthcheck, data)
+        self.assertEqual(data, data | healthcheck)
         data = {}
         self.scheduler.pod._set_container(
             'foo', 'bar', data, routable=True, build_type="buildpack", healthcheck={}
@@ -56,7 +56,7 @@ def test_set_container_applies_healthcheck_with_routable(self):
             'foo', 'bar', data,
             routable=False, healthcheck=healthcheck
         )
-        self.assertDictContainsSubset(healthcheck, data)
+        self.assertEqual(data, data | healthcheck)
         self.assertEqual(data.get('readinessProbe'), None)
 
         # now call without setting 'routable', should default to False
@@ -64,7 +64,7 @@ def test_set_container_applies_healthcheck_with_routable(self):
         self.scheduler.pod._set_container(
             'foo', 'bar', data, healthcheck=healthcheck
         )
-        self.assertDictContainsSubset(healthcheck, data)
+        self.assertEqual(data, data | healthcheck)
         self.assertEqual(data.get('readinessProbe'), None)
 
         data = {}
@@ -78,7 +78,7 @@ def test_set_container_applies_healthcheck_with_routable(self):
         self.scheduler.pod._set_health_checks(
             data, {'PORT': 80}, healthcheck=livenessProbe
         )
-        self.assertDictContainsSubset(healthcheck, data)
+        self.assertEqual(data, data | healthcheck)
         self.assertEqual(data.get('readinessProbe'), None)
 
     def test_set_container_limits(self):
@@ -88,9 +88,9 @@ def test_set_container_limits(self):
         data = {}
         self.scheduler.pod._set_container(
             'foo', 'bar', data, app_type='fake',
-            cpu={'fake': '500M'}, memory={'fake': '1024m'}
+            resources={"limits": {"cpu": "1", "memory": "64G"}},
         )
         # make sure CPU gets lower cased
-        self.assertEqual(data['resources']['limits']['cpu'], '500m', 'CPU should be lower cased')
+        self.assertEqual(data['resources']['limits']['cpu'], '1')
         # make sure first char of Memory is upper cased
-        self.assertEqual(data['resources']['limits']['memory'], '1024Mi', 'Memory should be upper cased')  # noqa
+        self.assertEqual(data['resources']['limits']['memory'], '64G')
diff --git a/rootfs/scheduler/tests/test_secrets.py b/rootfs/scheduler/tests/test_secrets.py
index faaddedec..8e000420a 100644
--- a/rootfs/scheduler/tests/test_secrets.py
+++ b/rootfs/scheduler/tests/test_secrets.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException, KubeException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class SecretsTest(TestCase):
@@ -114,16 +114,14 @@ def test_get_secret(self):
         self.assertEqual(response.status_code, 200, data)
         self.assertEqual(data['apiVersion'], 'v1')
         self.assertEqual(data['kind'], 'Secret')
-        self.assertDictContainsSubset(
-            {
-                'name': name,
-                'labels': {
-                    'app': self.namespace,
-                    'heritage': 'drycc'
-                }
-            },
-            data['metadata']
-        )
+        metadata = {
+            'name': name,
+            'labels': {
+                'app': self.namespace,
+                'heritage': 'drycc'
+            }
+        }
+        self.assertEqual(data['metadata'], data['metadata'] | metadata)
         self.assertEqual(data['data']['foo'], 'bar', data)
         self.assertEqual(data['data']['this'], 'that', data)
         self.assertEqual(data['type'], 'Opaque')
diff --git a/rootfs/scheduler/tests/test_services.py b/rootfs/scheduler/tests/test_services.py
index e769a954b..dd9ffe101 100644
--- a/rootfs/scheduler/tests/test_services.py
+++ b/rootfs/scheduler/tests/test_services.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class ServicesTest(TestCase):
@@ -128,14 +128,12 @@ def test_get_service(self):
         self.assertEqual(response.status_code, 200, data)
         self.assertEqual(data['apiVersion'], 'v1')
         self.assertEqual(data['kind'], 'Service')
-        self.assertDictContainsSubset(
-            {
-                'name': name,
-                'labels': {
-                    'app': self.namespace,
-                    'heritage': 'drycc'
-                }
-            },
-            data['metadata']
-        )
+        metadata = {
+            'name': name,
+            'labels': {
+                'app': self.namespace,
+                'heritage': 'drycc'
+            }
+        }
+        self.assertEqual(data['metadata'], data['metadata'] | metadata)
         self.assertEqual(data['spec']['ports'][0]['targetPort'], 5000)
diff --git a/rootfs/scheduler/tests/test_statefulsets.py b/rootfs/scheduler/tests/test_statefulsets.py
new file mode 100644
index 000000000..292b017a0
--- /dev/null
+++ b/rootfs/scheduler/tests/test_statefulsets.py
@@ -0,0 +1,57 @@
+"""
+Unit tests for the Drycc scheduler module.
+
+Run the tests with "./manage.py test scheduler"
+"""
+from scheduler.tests import TestCase
+from scheduler import KubeHTTPException
+
+
+class StatefulsetsTest(TestCase):
+    """Tests scheduler statefulset calls"""
+
+    def test_get_statefulsets(self):
+        response = self.scheduler.statefulset.get('drycc')
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertIn('items', data)
+        self.assertEqual(1, len(data['items']))
+        # simple verify of data
+        self.assertEqual(data['items'][0]['metadata']['name'], 'drycc')
+
+    def test_get_statefulset(self):
+        with self.assertRaises(
+            KubeHTTPException,
+            msg='failed to get Statefulset doesnotexist in Namespace drycc: 404 Not Found'  # noqa
+        ):
+            self.scheduler.statefulset.get('drycc', 'doesnotexist')
+        namespace = 'drycc'
+        name = 'drycc'
+        response = self.scheduler.statefulset.get(namespace, name)
+        data = response.json()
+        self.assertEqual(response.status_code, 200, data)
+        self.assertEqual(data['apiVersion'], 'apps/v1')
+        self.assertEqual(data['kind'], 'StatefulSet')
+        self.assertEqual(data['metadata']['name'], name)
+
+    def test_patch_statefulset(self):
+        namespace = 'drycc'
+        name = 'drycc'
+        manifest = {
+            'spec': {
+                'persistentVolumeClaimRetentionPolicy': {
+                    'whenScaled': 'Retain'
+                },
+                'replicas': 0
+            }
+        }
+        response = self.scheduler.statefulset.patch(
+            namespace,
+            name,
+            manifest=manifest,
+        )
+        when_scaled = response.json()["spec"]["persistentVolumeClaimRetentionPolicy"]['whenScaled']
+        self.assertEqual(
+            when_scaled,
+            'Retain'
+        )
diff --git a/rootfs/scheduler/tests/test_svcat.py b/rootfs/scheduler/tests/test_svcat.py
index 917fd357a..d61cdcd0f 100644
--- a/rootfs/scheduler/tests/test_svcat.py
+++ b/rootfs/scheduler/tests/test_svcat.py
@@ -5,7 +5,7 @@
 """
 from scheduler import KubeHTTPException
 from scheduler.tests import TestCase
-from scheduler.utils import generate_random_name
+from scheduler.tests.utils import generate_random_name
 
 
 class ServiceCatalogTest(TestCase):
diff --git a/rootfs/scheduler/tests/test_utils.py b/rootfs/scheduler/tests/test_utils.py
index 4b53eb013..cc965e7f9 100644
--- a/rootfs/scheduler/tests/test_utils.py
+++ b/rootfs/scheduler/tests/test_utils.py
@@ -1,5 +1,5 @@
 import unittest
-from scheduler import utils
+from api import utils
 
 
 class TestUtils(unittest.TestCase):
diff --git a/rootfs/scheduler/utils.py b/rootfs/scheduler/tests/utils.py
similarity index 80%
rename from rootfs/scheduler/utils.py
rename to rootfs/scheduler/tests/utils.py
index ed0f273f3..90cf6fdf8 100644
--- a/rootfs/scheduler/utils.py
+++ b/rootfs/scheduler/tests/utils.py
@@ -1,4 +1,3 @@
-from copy import deepcopy
 import random
 
 
@@ -60,34 +59,3 @@ def generate_random_name():
     ]
     return "{}-{}".format(
         random.choice(adjectives), random.choice(nouns))
-
-
-def dict_merge(origin, merge):
-    """
-    Recursively merges dict's. not just simple a["key"] = b["key"], if
-    both a and b have a key who's value is a dict then dict_merge is called
-    on both values and the result stored in the returned dictionary.
-    Also handles merging lists if they occur within the dict
-    """
-    if not isinstance(merge, dict):
-        return merge
-
-    result = deepcopy(origin)
-    for key, value in merge.items():
-        if key in result and isinstance(result[key], dict):
-            result[key] = dict_merge(result[key], value)
-        else:
-            if isinstance(value, list):
-                if key not in result:
-                    result[key] = value
-                else:
-                    # merge lists without leaving potential duplicates
-                    # result[key] = list(set(result[key] + value))  # changes the order as well
-                    for item in value:
-                        if item in result[key]:
-                            continue
-
-                        result[key].append(item)
-            else:
-                result[key] = deepcopy(value)
-    return result
