feat(ingress): Improving the configuration of ingress

Author: duanhongyi

charts/controller/templates/controller-certificate.yaml

@@ -0,0 +1,20 @@
+{{ if .Values.cert_manager_enabled }}
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: Certificate
+metadata:
+  name: drycc-controller
+  namespace: "{{ .Release.Namespace }}"
+spec:
+  secretName: drycc-controller-auto-tls
+  issuerRef:
+    name: drycc-letsencrypt
+    kind: ClusterIssuer
+  dnsNames:
+  - drycc.{{ .Values.platform_domain }}
+  acme:
+    config:
+    - http01:
+        ingressClass: "{{ .Values.global.ingress_class }}"
+      domains:
+      - drycc.{{ .Values.platform_domain }}
+{{- end }}

charts/controller/templates/controller-cluster-issuer.yaml

@@ -0,0 +1,15 @@
+apiVersion: certmanager.k8s.io/v1alpha1
+kind: ClusterIssuer
+metadata:
+  name: drycc-letsencrypt
+spec:
+  acme:
+    # The ACME server URL
+    server: https://acme-v02.api.letsencrypt.org/directory
+    # Email address used for ACME registration
+    email: "{{ .Values.global.email }}"
+    # Name of a secret used to store the ACME account private key
+    privateKeySecretRef:
+      name: drycc-letsencrypt
+    # Enable HTTP01 validations
+    http01: {}

charts/controller/templates/controller-ingress.yaml

@@ -1,4 +1,3 @@
-{{ if not (eq .Values.global.ingress_class "") }}
 apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
@@ -25,4 +24,9 @@ spec:
         backend:
           serviceName: drycc-controller
           servicePort: 80
-{{- end }}
+  {{ if .Values.cert_manager_enabled }}
+  tls:
+    - secretName: drycc-controller-auto-tls
+      hosts:
+        - drycc.{{ .Values.platform_domain }}
+  {{- end }}

charts/controller/values.yaml

@@ -19,8 +19,12 @@ k8s_api_verify_tls: "true"
 #
 # This will be the hostname that is used to build endpoints such as "drycc.$HOSTNAME"
 platform_domain: ""
+# Whether cert_manager is enabled to automatically generate controller certificates
+cert_manager_enabled: "true"
 
 global:
+  # Admin email, used for each component to send email to administrator
+  email: "drycc@drycc.cc"
   # Set the storage backend
   #
   # Valid values are:
@@ -52,8 +56,5 @@ global:
   secret_prefix: "private-registry"
   # Role-Based Access Control for Kubernetes >= 1.5
   use_rbac: false
-  # Experimental feature to use Kubernetes ingress instead of Workflow's drycc-router.
-  # If ingress_class is empty, use drycc-router
-  # Otherwise, please check `kubernetes.io/ingress.class`
-  # The cert-manager component must be installed
+  # Please check `kubernetes.io/ingress.class`
   ingress_class: ""

rootfs/api/models/app.py

@@ -237,9 +237,10 @@ def create(self, *args, **kwargs):  # noqa
                     self._scheduler.ingress.get(ingress)
                 except KubeException:
                     self.log("creating Ingress {}".format(namespace), level=logging.INFO)
+                    host = "%s.%s" % (ingress, settings.PLATFORM_DOMAIN)
                     self._scheduler.ingress.create(
                         ingress, settings.INGRESS_CLASS, namespace,
-                        settings.PLATFORM_DOMAIN)
+                        hosts=[host, ], tls=[])
         except KubeException as e:
             raise ServiceUnavailable('Could not create Ingress in Kubernetes') from e
         try:

rootfs/api/settings/production.py

@@ -262,6 +262,7 @@
 
 # experimental native ingress
 INGRESS_CLASS = os.environ.get('DRYCC_INGRESS_CLASS', '')
+
 PLATFORM_DOMAIN = os.environ.get('DRYCC_PLATFORM_DOMAIN', '')
 
 # k8s image policies

rootfs/scheduler/resources/certificate.py

@@ -0,0 +1,55 @@
+import json
+from scheduler.resources import Resource
+from scheduler.exceptions import KubeHTTPException, KubeException
+
+
+class Certificate(Resource):
+
+    def manifest(self, namespace, name, ingress_class, hosts):
+        data = {
+            "apiVersion": "certmanager.k8s.io/v1alpha1",
+            "kind": "Certificate",
+            "metadata": {
+                "name": name,
+                "namespace": namespace
+            },
+            "spec": {
+                "secretName": "%s-auto-tls" % name,
+                "issuerRef": {
+                    "name": "drycc-controller-letsencrypt",
+                    "kind": "ClusterIssuer"
+                },
+                "dnsNames": hosts,
+                "acme": {
+                    "config": [
+                        {
+                            "http01": {
+                                "ingressClass": ingress_class
+                            },
+                            "domains": hosts
+                        }
+                    ]
+                }
+            }
+        }
+        return data
+
+    def get(self, namespace, name):
+        """
+        Fetch a single Ingress or a list of Ingresses
+        """
+        if name is not None:
+            url = "/apis/certmanager/v1alpha1/namespaces/%s/certificates/%s" % (namespace, name)
+            message = 'get Ingress ' + name
+        else:
+            url = "/apis/certmanager/v1alpha1/namespaces/%s/certificates" % namespace
+            message = 'get Ingresses'
+
+    def create(self, namespace, name, ingress_class, hosts):
+        pass
+
+    def put(self, namespace, name, ingress_class, hosts):
+        pass
+
+    def delete(self, namespace, name):
+        pass

rootfs/scheduler/resources/ingress.py

@@ -1,10 +1,95 @@
 from scheduler.exceptions import KubeHTTPException
 from scheduler.resources import Resource
 
+MANIFEAT_CLASSES = {}
+
+
+class BaseManifest(object):
+
+    def manifest(self, ingress, ingress_class, namespace, **kwargs):
+        path = "/*" if ingress_class in ("gce", "alb") else "/"
+        hosts, tls = kwargs.pop("hosts"), kwargs.pop("tls")
+        data = {
+            "kind": "Ingress",
+            "apiVersion": "extensions/v1beta1",
+            "metadata": {
+                "name": ingress,
+                "annotations": {
+                    "kubernetes.io/tls-acme": "true",
+                    "kubernetes.io/ingress.class": ingress_class
+                }
+            },
+            "spec": {
+                "rules": [{
+                    "host": host,
+                    "http": {
+                        "paths": [
+                            {
+                                "path": path,
+                                "backend": {
+                                    "serviceName": ingress,
+                                    "servicePort": 80
+                                }
+                            }
+                        ]
+                    }
+                } for host in hosts],
+                "tls": tls
+            }
+        }
+        return data
+MANIFEAT_CLASSES["default"] = BaseManifest
+
+
+class NginxManifest(BaseManifest):
+
+    def manifest(self, ingress, ingress_class, namespace, **kwargs):
+        data = BaseManifest.manifest(
+            self, ingress, ingress_class, namespace, **kwargs)
+        if "whitelist" in kwargs:
+            whitelist = ", ".join(kwargs.pop("whitelist"))
+            data.update({
+                "nginx.ingress.kubernetes.io/whitelist-source-range": whitelist
+            })
+        if "ssl_redirect" in kwargs:
+            ssl_redirect = kwargs.pop("ssl_redirect")
+            data.update({
+                "nginx.ingress.kubernetes.io/ssl-redirect": ssl_redirect
+            })
+        return data
+MANIFEAT_CLASSES["nginx"] = NginxManifest
+
+
+class TraefikManifest(BaseManifest):
+
+    def manifest(self, ingress, ingress_class, namespace, **kwargs):
+        data = BaseManifest.manifest(
+            self, ingress, ingress_class, namespace, **kwargs)
+        if "whitelist" in kwargs:
+            whitelist = ", ".join(kwargs.pop("whitelist"))
+            data.update({
+                "ingress.kubernetes.io/whitelist-x-forwarded-for": "true",
+                "traefik.ingress.kubernetes.io/whitelist-source-range": whitelist
+            })
+        if "ssl_redirect" in kwargs:
+            ssl_redirect = kwargs.pop("ssl_redirect")
+            data.update({
+                "ingress.kubernetes.io/ssl-redirect": ssl_redirect
+            })
+        return data
+MANIFEAT_CLASSES["traefik"] = TraefikManifest
+
 
 class Ingress(Resource):
     short_name = 'ingress'
 
+    def manifest(self, ingress, ingress_class, namespace, **kwargs):
+        if ingress_class not in MANIFEAT_CLASSES:
+            ingress_class = "default"
+        return MANIFEAT_CLASSES.get(ingress_class)().manifest(
+            ingress, ingress_class, namespace, **kwargs
+        )
+
     def get(self, name=None, **kwargs):
         """
         Fetch a single Ingress or a list of Ingresses
@@ -22,51 +107,30 @@ def get(self, name=None, **kwargs):
 
         return response
 
-    def create(self, ingress, ingress_class, namespace, hostname):
+    def create(self, ingress, ingress_class, namespace, **kwargs):
         url = "/apis/extensions/v1beta1/namespaces/%s/ingresses" % namespace
-        path = "/*" if ingress_class in ("gce", "alb") else "/"
-        data = {
-            "kind": "Ingress",
-            "apiVersion": "extensions/v1beta1",
-            "metadata": {
-                "name": ingress,
-                "annotations": {
-                    "kubernetes.io/tls-acme": "true"
-                }
-            },
-            "spec": {
-                "rules": [
-                    {
-                        "host": ingress + "." + hostname,
-                        "http": {
-                            "paths": [
-                                {
-                                    "path": path,
-                                    "backend": {
-                                        "serviceName": ingress,
-                                        "servicePort": 80
-                                    }
-                                }
-                            ]
-                        }
-                    }
-                ]
-            }
-        }
-        if ingress_class:
-            data["metadata"]["annotations"].update({
-                "kubernetes.io/ingress.class": ingress_class})
+        data = self.manifest(ingress, ingress_class, namespace, **kwargs)
         response = self.http_post(url, json=data)
 
         if not response.status_code == 201:
             raise KubeHTTPException(response, "create Ingress {}".format(namespace))
 
         return response
 
+    def update(self, ingress, ingress_class, namespace, **kwargs):
+        url = "/apis/extensions/v1beta1/namespaces/%s/ingresses/%s" % (namespace, ingress)
+        data = self.manifest(ingress, ingress_class, namespace, **kwargs)
+        response = self.http_put(url, json=data)
+
+        if self.unhealthy(response.status_code):
+            raise KubeHTTPException(response, "update Ingress {}".format(namespace))
+
+        return response
+
     def delete(self, namespace, ingress):
         url = "/apis/extensions/v1beta1/namespaces/%s/ingresses/%s" % (namespace, ingress)
         response = self.http_delete(url)
         if self.unhealthy(response.status_code):
             raise KubeHTTPException(response, 'delete Ingress "{}"', namespace)
 
-        return response
+        return response

rootfs/scheduler/resources/secret.py

@@ -113,3 +113,55 @@ def delete(self, namespace, name):
             )
 
         return response
+
+
+class Certificate(Resource):
+
+    def manifest(self, namespace, name, ingress_class, hosts):
+        data = {
+            "apiVersion": "certmanager.k8s.io/v1alpha1",
+            "kind": "Certificate",
+            "metadata": {
+                "name": name,
+                "namespace": namespace
+            },
+            "spec": {
+                "secretName": "%s-auto-tls" % name,
+                "issuerRef": {
+                    "name": "drycc-controller-letsencrypt",
+                    "kind": "ClusterIssuer"
+                },
+                "dnsNames": hosts,
+                "acme": {
+                    "config": [
+                        {
+                            "http01": {
+                                "ingressClass": ingress_class
+                            },
+                            "domains": hosts
+                        }
+                    ]
+                }
+            }
+        }
+        return data
+
+    def get(self, namespace, name):
+        """
+        Fetch a single Ingress or a list of Ingresses
+        """
+        if name is not None:
+            url = "/apis/certmanager/v1alpha1/namespaces/%s/certificates/%s" % (namespace, name)
+            message = 'get Ingress ' + name
+        else:
+            url = "/apis/certmanager/v1alpha1/namespaces/%s/certificates" % namespace
+            message = 'get Ingresses'
+
+    def create(self, namespace, name, ingress_class, hosts):
+        pass
+
+    def put(self, namespace, name, ingress_class, hosts):
+        pass
+
+    def delete(self, namespace, name):
+        pass

rootfs/scheduler/tests/test_ingress.py

@@ -13,7 +13,8 @@ class IngressTest(TestCase):
     def test_create_ingress(self):
         # Ingress assumes that the namespace and ingress name are always the same
         self.scheduler.ns.create("test-ingress")
-        self.scheduler.ingress.create("test-ingress", "nginx", "test-ingress", "test-ingress")
+        self.scheduler.ingress.create("test-ingress", "nginx", "test-ingress",
+            hosts=["test-ingress"], tls=[])
 
     def test_get_ingresses(self):
         response = self.scheduler.ingress.get()