feat(controller): add admission webhook

duanhongyi · duanhongyi · commit f279fc6a5c4b · 2021-11-24T10:58:33.000+08:00
diff --git a/charts/controller/templates/_helpers.tpl b/charts/controller/templates/_helpers.tpl
@@ -32,6 +32,8 @@ env:
 {{- end }}
 - name: "TZ"
   value: {{ .Values.time_zone | default "UTC" | quote }}
+- name: "DJANGO_SETTINGS_MODULE"
+  value: "api.settings.production"
 {{- if (.Values.deploy_hook_urls) }}
 - name: DRYCC_DEPLOY_HOOK_URLS
   value: "{{ .Values.deploy_hook_urls }}"
diff --git a/charts/controller/templates/controller-cronjob-daily.yaml b/charts/controller/templates/controller-cronjob-daily.yaml
@@ -32,7 +32,7 @@ spec:
           containers:
           - image: {{.Values.image_registry}}/{{.Values.image_org}}/controller:{{.Values.image_tag}}
             imagePullPolicy: {{.Values.image_pull_policy}}
-            name: drycc-controller-load_db_state_to_k8s
+            name: drycc-controller-load-db-state-to-k8s
             command:
               - /bin/bash
               - -c
diff --git a/charts/controller/templates/controller-webhook-deloyment.yaml b/charts/controller/templates/controller-webhook-deloyment.yaml
@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: drycc-controller-webhook
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: drycc-controller-webhook
+  template:
+    metadata:
+      labels:
+        component: drycc-controller-webhook
+    spec:
+      initContainers:
+      - name: drycc-controller-init
+        image: docker.io/drycc/python-dev:latest
+        imagePullPolicy: {{.Values.image_pull_policy}}
+        command:
+          - netcat
+          - -v
+          - -u
+          - $(DRYCC_DATABASE_URL),$(DRYCC_RABBITMQ_URL)
+          - -a
+          - $(DRYCC_REDIS_ADDRS)
+        {{- include "controller.envs" . | indent 8 }}
+      containers:
+      - name: drycc-controller
+        image: {{.Values.image_registry}}/{{.Values.image_org}}/controller:{{.Values.image_tag}}
+        imagePullPolicy: {{.Values.image_pull_policy}}
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 8443
+            scheme: HTTPS
+          initialDelaySeconds: 30
+          timeoutSeconds: 10
+          periodSeconds: 5
+        ports:
+          - containerPort: 8443
+            name: https
+        volumeMounts:
+        - name: drycc-controller-webhook-cert
+          mountPath: /etc/controller/webhook/cert
+        {{- include "controller.limits" . | indent 8 }}
+        {{- include "controller.envs" . | indent 8 }}
+      volumes:
+      - name: drycc-controller-webhook-cert                                                                                                                                                                                               
+        secret:                                                                                                                                                                                                                           
+          secretName: drycc-controller-webhook-cert                                                                                                                                                                                       
+          items:  
+          - key: token                                                                                                                                                                                                                  
+            path: token                                                                                                                                                                                                                           
+          - key: tls.crt                                                                                                                                                                                                                  
+            path: tls.crt                                                                                                                                                                                                                 
+          - key: tls.key                                                                                                                                                                                                                  
+            path: tls.key
diff --git a/charts/controller/templates/controller-webhook-register.yaml b/charts/controller/templates/controller-webhook-register.yaml
@@ -0,0 +1,40 @@
+{{- $token := randAlphaNum 128 | lower }}
+{{- $ca := genCA "controller-webhook-ca" 3650 }}
+{{- $altName1 := printf "drycc-controller-webhook.%s" .Release.Namespace }}
+{{- $altName2 := printf "drycc-controller-webhook.%s.svc" .Release.Namespace }}
+{{- $cert := genSignedCert "drycc-controller-webhook" nil (list $altName1 $altName2) 3650 $ca }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: drycc-controller-webhook
+webhooks:
+- name: "{{ $altName2 }}"
+  sideEffects: None
+  admissionReviewVersions: ["v1"]
+  clientConfig:
+    caBundle: {{ b64enc $ca.Cert }}
+    service:
+      name: drycc-controller-webhook
+      namespace: "{{ .Release.Namespace }}"
+      path: "{{ printf "/v2/webhooks/scale/%s/" $token }}"
+      port: 8443
+  failurePolicy: Fail
+  rules:
+  - operations: ["UPDATE"]
+    apiGroups: ["apps"]
+    apiVersions: ["*"]
+    resources: ["deployments/scale"]
+  timeoutSeconds: 30
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: drycc-controller-webhook-cert
+  labels:
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+type: Opaque
+data:
+  token: {{ b64enc $token }}
+  tls.crt: {{ b64enc $cert.Cert }}
+  tls.key: {{ b64enc $cert.Key }}
diff --git a/charts/controller/templates/controller-webhook-service.yaml b/charts/controller/templates/controller-webhook-service.yaml
@@ -0,0 +1,11 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: drycc-controller-webhook
+spec:
+  selector:
+    component: drycc-controller-webhook
+  ports:
+  - name: https
+    port: 8443
+    targetPort: 8443
diff --git a/rootfs/api/models/app.py b/rootfs/api/models/app.py
@@ -473,18 +473,16 @@ def scale(self, user, structure):  # noqa
         new_structure.update(structure)
 
         if new_structure != self.structure:
-            # save new structure to the database
-            self.structure = new_structure
-            self.procfile_structure = release.build.procfile
-            self.save()
-
             try:
                 self._scale_pods(structure)
             except ServiceUnavailable:
                 # scaling failed, go back to old scaling numbers
                 self._scale_pods(old_structure)
                 raise
-
+            # save new structure to the database
+            self.structure = new_structure
+            self.procfile_structure = release.build.procfile
+            self.save()
             msg = '{} scaled pods '.format(user.username) + ' '.join(
                 "{}={}".format(k, v) for k, v in list(structure.items()))
             self.log(msg)
diff --git a/rootfs/api/settings/celery.py b/rootfs/api/settings/celery.py
@@ -22,7 +22,6 @@ class Config:
     task_default_queue = 'priority.low'
 
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.settings.production')
 app = Celery('drycc')
 app.config_from_object(Config)
 app.conf.update(
diff --git a/rootfs/api/settings/production.py b/rootfs/api/settings/production.py
@@ -24,7 +24,8 @@
 # https://docs.djangoproject.com/en/1.11/ref/checks/#security
 SILENCED_SYSTEM_CHECKS = [
     'security.W004',
-    'security.W008'
+    'security.W008',
+    'security.W012',
 ]
 
 CONN_MAX_AGE = 60 * 3
@@ -81,7 +82,7 @@
     'corsheaders.middleware.CorsMiddleware',
     'django.middleware.security.SecurityMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    # 'django.middleware.csrf.CsrfViewMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.middleware.common.CommonMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
@@ -138,9 +139,10 @@
 )
 
 X_FRAME_OPTIONS = 'DENY'
-# CSRF_COOKIE_SECURE = True
-CSRF_COOKIE_HTTPONLY = True
-# SESSION_COOKIE_SECURE = True
+CSRF_COOKIE_SECURE = True
+CSRF_COOKIE_HTTPONLY = False
+CSRF_COOKIE_SAMESITE = None
+SESSION_COOKIE_SECURE = False
 SECURE_CONTENT_TYPE_NOSNIFF = True
 SECURE_BROWSER_XSS_FILTER = True
 
@@ -478,4 +480,11 @@
 # Workflow-manager Configuration Options
 WORKFLOW_MANAGER_URL = os.environ.get('DRYCC_WORKFLOW_MANAGER_URL', None)
 WORKFLOW_MANAGER_ACCESS_KEY = os.environ.get('WORKFLOW_MANAGER_ACCESS_KEY', None)
-WORKFLOW_MANAGER_SECRET_KEY = os.environ.get('WORKFLOW_MANAGER_SECRET_KEY', None)
+WORKFLOW_MANAGER_SECRET_KEY = os.environ.get('WORKFLOW_MANAGER_SECRET_KEY', None)
+
+# Drycc admission webhook token
+if os.path.exists("/etc/controller/webhook/cert"):
+    with open("/etc/controller/webhook/cert/token") as f:
+        DRYCC_ADMISSION_WEBHOOK_TOKEN = f.read()
+else:
+    DRYCC_ADMISSION_WEBHOOK_TOKEN = None
diff --git a/rootfs/api/settings/testing.py b/rootfs/api/settings/testing.py
@@ -1,10 +1,16 @@
 import random
 import string
 import os
-
+from django.core import signals
+from api.settings.celery import app
 from api.settings.production import DATABASES
 from api.settings.production import *  # noqa
 
+# Monkey patch celery
+app.conf.update(task_always_eager=True)
+signals.request_started.send = lambda sender, **named: []
+signals.request_finished.send = lambda sender, **named: []
+
 # A boolean that turns on/off debug mode.
 # https://docs.djangoproject.com/en/1.11/ref/settings/#debug
 DEBUG = True
diff --git a/rootfs/api/tasks.py b/rootfs/api/tasks.py
@@ -7,6 +7,7 @@
 from celery import shared_task
 
 from api import manager
+from api.models import ServiceUnavailable
 from api.models.resource import Resource
 logger = logging.getLogger(__name__)
 
@@ -46,3 +47,17 @@ def send_measurements(measurements: List[Dict[str, str]]):
         measurement.post(measurements)
     finally:
         signals.request_finished.send(sender=task_id)
+
+
+@shared_task(
+    autoretry_for=(ServiceUnavailable, ),
+    retry_jitter=True,
+    retry_kwargs={'max_retries': 3}
+)
+def scale_app(app, user, structure):
+    task_id = uuid.uuid4().hex
+    signals.request_started.send(sender=task_id)
+    try:
+        app.scale(user, structure)
+    finally:
+        signals.request_finished.send(sender=task_id)
diff --git a/rootfs/api/tests/__init__.py b/rootfs/api/tests/__init__.py
@@ -54,6 +54,7 @@ def run_tests(self, test_labels, extra_tests=None, **kwargs):
 
 
 class DryccTestCase(APITestCase):
+
     def create_app(self, name=None):
         body = {}
         if name:
diff --git a/rootfs/api/tests/test_pods.py b/rootfs/api/tests/test_pods.py
@@ -298,12 +298,13 @@ def test_container_errors(self, mock_requests):
         url = "/v2/apps/{app_id}/scale".format(**locals())
         body = {'web': 'not_an_int'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {'detail': "Invalid scaling format: invalid literal for "
-                                                   "int() with base 10: 'not_an_int'"})
+        self.assertEqual(response.status_code, 204, response.data)
+        self.assertEqual(app.structure, {})
+
         body = {'invalid': 1}
         response = self.client.post(url, body)
-        self.assertContains(response, 'Container type invalid', status_code=404)
+        self.assertEqual(response.status_code, 204, response.data)
+        self.assertEqual(app.structure, {})
 
     def test_container_str(self, mock_requests):
         """Test the text representation of a container."""
@@ -408,40 +409,49 @@ def test_scale_errors(self, mock_requests):
         url = "/v2/apps/{app_id}/scale".format(**locals())
         body = {'web': -1}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {"detail": "Invalid scaling format: "
-                                         "[ErrorDetail(string='Must be greater "
-                                         "than or equal to zero', code='invalid')]"})
+        self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.structure["web"], 1)
 
         # scale to something other than a number
         url = "/v2/apps/{app_id}/scale".format(**locals())
         body = {'web': 'one'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(response.status_code, 204, response.data)
+        self.assertEqual(app.structure["web"], 1)
 
         # scale to something other than a number
         url = "/v2/apps/{app_id}/scale".format(**locals())
         body = {'web': [1]}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
+        self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.structure["web"], 1)
 
         # scale with a non-existent proc type
         url = "/v2/apps/{app_id}/scale".format(**locals())
         body = {'foo': 1}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 404, response.data)
+        self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual("foo" in app.structure, False)
 
         # scale up to an integer as a sanity check
         url = "/v2/apps/{app_id}/scale".format(**locals())
         body = {'web': 1}
         response = self.client.post(url, body)
         self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual(app.structure["web"], 1)
 
         with mock.patch('scheduler.KubeHTTPClient.scale') as mock_kube:
             mock_kube.side_effect = KubeException('Boom!')
             url = "/v2/apps/{app_id}/scale".format(**locals())
             response = self.client.post(url, {'web': 10})
-            self.assertEqual(response.status_code, 503, response.data)
+            self.assertEqual(response.status_code, 204, response.data)
+            app = App.objects.get(id=app_id)
+            self.assertEqual(app.structure["web"], 1)
 
     def test_admin_can_manage_other_pods(self, mock_requests):
         """If a non-admin user creates a container, an administrator should be able to
@@ -483,8 +493,9 @@ def test_scale_without_build_should_error(self, mock_requests):
         url = '/v2/apps/{app_id}/scale'.format(**locals())
         body = {'web': '1'}
         response = self.client.post(url, body)
-        self.assertEqual(response.status_code, 400, response.data)
-        self.assertEqual(response.data, {'detail': 'No build associated with this release'})
+        self.assertEqual(response.status_code, 204, response.data)
+        app = App.objects.get(id=app_id)
+        self.assertEqual("web" in app.structure, False)
 
     def test_command_good(self, mock_requests):
         """Test the default command for each container workflow"""
diff --git a/rootfs/api/urls.py b/rootfs/api/urls.py
@@ -1,8 +1,6 @@
 """
 URL routing patterns for the Drycc REST API.
 """
-
-
 from django.conf import settings
 from django.conf.urls import include, url
 from rest_framework.routers import DefaultRouter
@@ -12,16 +10,17 @@
 
 router = DefaultRouter(trailing_slash=False)
 extra = getattr(settings, setting_name('TRAILING_SLASH'), True) and '/' or ''
+
 # Add the generated REST URLs and login/logout endpoint
-urlpatterns = [
+app_urlpatterns = [
     url(r'^', include(router.urls)),
     url(r'^login/(?P<backend>[^/]+){0}$'.format(extra), views.auth,
         name='begin'),
     url(r'^complete/(?P<backend>[^/]+){0}$'.format(extra), views.complete,
         name='complete'),
     url('', include('social_django.urls', namespace='social')),
-    url(r'auth/login/?$', views.AuthLoginView.as_view()),
-    url(r'auth/token/(?P<key>[-_\w]+)/?$', views.AuthTokenView.as_view()),
+    url(r'auth/login/?$', views.AuthLoginView.as_view({"post": "login"})),
+    url(r'auth/token/(?P<key>[-_\w]+)/?$', views.AuthTokenView.as_view({"get": "token"})),
     # application release components
     url(r"^apps/(?P<id>{})/config/?$".format(settings.APP_URL_REGEX),
         views.ConfigViewSet.as_view({'get': 'retrieve', 'post': 'create'})),
@@ -152,3 +151,13 @@
     url(r'^apps/(?P<id>{})/metrics/(?P<container_type>[a-z0-9]+(\-[a-z0-9]+)*)/?$'.format(settings.APP_URL_REGEX),  # noqa
         views.MetricView.as_view({'get': 'status'})),
 ]
+
+webhook_urlpatterns = [
+    url(r'^webhooks/scale/(?P<token>.+)/?$', views.AdmissionWebhook.as_view({'post': 'scale'})),
+]
+
+# If there is a mutating admission webhook configuration, use webhook url
+if settings.DRYCC_ADMISSION_WEBHOOK_TOKEN:
+    urlpatterns = webhook_urlpatterns
+else:
+    urlpatterns = app_urlpatterns
diff --git a/rootfs/api/views.py b/rootfs/api/views.py
diff --git a/rootfs/bin/boot b/rootfs/bin/boot
diff --git a/rootfs/bin/test-unit b/rootfs/bin/test-unit
diff --git a/rootfs/drycc/gunicorn/config.py b/rootfs/drycc/gunicorn/config.py
diff --git a/rootfs/requirements.txt b/rootfs/requirements.txt
