Merge pull request #71 from notmaxx/feature_services

feat(controller): extra services support

Author: Cryptophobia

rootfs/api/management/commands/load_db_state_to_k8s.py

@@ -1,7 +1,7 @@
 from django.core.management.base import BaseCommand
 from django.shortcuts import get_object_or_404
 
-from api.models import Key, App, Domain, Certificate
+from api.models import Key, App, Domain, Certificate, Service
 from api.exceptions import DeisException, AlreadyExists
 
 
@@ -52,7 +52,7 @@ def save_apps(self):
             except DeisException as error:
                 print('ERROR: Problem saving to model {} for {}'
                       'due to {}'.format(str(App.__name__), str(app), str(error)))
-        for model in (Key, Domain, Certificate):
+        for model in (Key, Domain, Certificate, Service):
             for obj in model.objects.all():
                 try:
                     obj.save()

rootfs/api/migrations/0027_auto_20180424_1742.py

@@ -0,0 +1,38 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.4 on 2018-04-24 17:42
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('api', '0026_release_exception'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Service',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('procfile_type', models.TextField()),
+                ('path_pattern', models.TextField()),
+                ('app', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.App')),
+                ('owner', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                'ordering': ['-created'],
+                'get_latest_by': 'created',
+            },
+        ),
+        migrations.AlterUniqueTogether(
+            name='service',
+            unique_together=set([('app', 'procfile_type')]),
+        ),
+    ]

rootfs/api/models/__init__.py

@@ -72,10 +72,12 @@ def _scheduler(self):
         mod = importlib.import_module(settings.SCHEDULER_MODULE)
         return mod.SchedulerClient(settings.SCHEDULER_URL, settings.K8S_API_VERIFY_TLS)
 
-    def _fetch_service_config(self, app):
+    def _fetch_service_config(self, app, svc_name=None):
         try:
             # Get the service from k8s to attach the domain correctly
-            svc = self._scheduler.svc.get(app, app).json()
+            if svc_name is None:
+                svc_name = app
+            svc = self._scheduler.svc.get(app, svc_name).json()
         except KubeException as e:
             raise ServiceUnavailable('Could not fetch Kubernetes Service {}'.format(app)) from e
 
@@ -90,9 +92,9 @@ def _fetch_service_config(self, app):
 
         return svc
 
-    def _load_service_config(self, app, component):
+    def _load_service_config(self, app, component, svc_name=None):
         # fetch setvice definition with minimum structure
-        svc = self._fetch_service_config(app)
+        svc = self._fetch_service_config(app, svc_name)
 
         # always assume a .deis.io/ ending
         component = "%s.deis.io/" % component
@@ -103,9 +105,11 @@ def _load_service_config(self, app, component):
 
         return config
 
-    def _save_service_config(self, app, component, data):
+    def _save_service_config(self, app, component, data, svc_name=None):
+        if svc_name is None:
+            svc_name = app
         # fetch setvice definition with minimum structure
-        svc = self._fetch_service_config(app)
+        svc = self._fetch_service_config(app, svc_name)
 
         # always assume a .deis.io ending
         component = "%s.deis.io/" % component
@@ -116,7 +120,7 @@ def _save_service_config(self, app, component, data):
 
         # Update the k8s service for the application with new service information
         try:
-            self._scheduler.svc.update(app, app, svc)
+            self._scheduler.svc.update(app, svc_name, svc)
         except KubeException as e:
             raise ServiceUnavailable('Could not update Kubernetes Service {}'.format(app)) from e
 
@@ -142,6 +146,7 @@ class Meta:
 from .certificate import Certificate, validate_certificate  # noqa
 from .config import Config  # noqa
 from .domain import Domain  # noqa
+from .service import Service  # noqa
 from .key import Key, validate_base64  # noqa
 from .release import Release  # noqa
 from .tls import TLS  # noqa

rootfs/api/models/app.py

@@ -892,6 +892,10 @@ def maintenance_mode(self, mode):
             self._scheduler.svc.update(self.id, self.id, data=old_service)
             raise ServiceUnavailable(str(e)) from e
 
+        # set maintenance mode for services
+        for svc in self.service_set.all():
+            svc.maintenance_mode(mode)
+
     def routable(self, routable):
         """
         Turn on/off if an application is publically routable

rootfs/api/models/service.py

@@ -0,0 +1,145 @@
+import logging
+
+from django.db import models
+from django.conf import settings
+
+from api.models import AuditedModel, ServiceUnavailable
+from scheduler import KubeException
+
+logger = logging.getLogger(__name__)
+
+
+class Service(AuditedModel):
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.PROTECT)
+    app = models.ForeignKey('App', on_delete=models.CASCADE)
+    procfile_type = models.TextField(blank=False, null=False, unique=False)
+    path_pattern = models.TextField(blank=False, null=False, unique=False)
+
+    class Meta:
+        get_latest_by = 'created'
+        unique_together = (('app', 'procfile_type'))
+        ordering = ['-created']
+
+    def __str__(self):
+        return self._svc_name()
+
+    def as_dict(self):
+        return {
+            "procfile_type": self.procfile_type,
+            "path_pattern": self.path_pattern
+        }
+
+    def create(self, *args, **kwargs):  # noqa
+        # create required minimum service in k8s for the application
+        namespace = self._namespace()
+        svc_name = self._svc_name()
+        self.log('creating Service: {}'.format(svc_name), level=logging.DEBUG)
+        try:
+            try:
+                self._scheduler.svc.get(namespace, svc_name)
+            except KubeException:
+                self._scheduler.svc.create(namespace, svc_name)
+        except KubeException as e:
+            raise ServiceUnavailable('Kubernetes service could not be created') from e
+        # config service
+        annotations = self._gather_settings()
+        routable = annotations.pop('routable')
+        self._update_service(namespace, self.procfile_type, routable, annotations)
+
+    def save(self, *args, **kwargs):
+        service = super(Service, self).save(*args, **kwargs)
+
+        self.create()
+
+        return service
+
+    def delete(self, *args, **kwargs):
+        namespace = self._namespace()
+        svc_name = self._svc_name()
+        self.log('deleting Service: {}'.format(svc_name), level=logging.DEBUG)
+        try:
+            self._scheduler.svc.delete(namespace, svc_name)
+        except KubeException:
+            # swallow exception
+            # raise ServiceUnavailable('Kubernetes service could not be deleted') from e
+            self.log('Kubernetes service cannot be deleted: {}'.format(svc_name),
+                     level=logging.ERROR)
+
+        # Delete from DB
+        return super(Service, self).delete(*args, **kwargs)
+
+    def log(self, message, level=logging.INFO):
+        """Logs a message in the context of this service.
+
+        This prefixes log messages with an application "tag" that the customized deis-logspout will
+        be on the lookout for.  When it's seen, the message-- usually an application event of some
+        sort like releasing or scaling, will be considered as "belonging" to the application
+        instead of the controller and will be handled accordingly.
+        """
+        logger.log(level, "[{}]: {}".format(self.id, message))
+
+    def maintenance_mode(self, mode):
+        """
+        Turn service maintenance mode on/off
+        """
+        namespace = self._namespace()
+        svc_name = self._svc_name()
+
+        try:
+            service = self._fetch_service_config(namespace, svc_name)
+        except (ServiceUnavailable, KubeException) as e:
+            # ignore non-existing services
+            return
+
+        old_service = service.copy()  # in case anything fails for rollback
+
+        try:
+            service['metadata']['annotations']['router.deis.io/maintenance'] = str(mode).lower()
+            self._scheduler.svc.update(namespace, svc_name, data=service)
+        except KubeException as e:
+            self._scheduler.svc.update(namespace, svc_name, data=old_service)
+            raise ServiceUnavailable(str(e)) from e
+
+    def _namespace(self):
+        return self.app.id
+
+    def _svc_name(self):
+        return "{}-{}".format(self.app.id, self.procfile_type)
+
+    def _gather_settings(self):
+        app_settings = self.app.appsettings_set.latest()
+        return {
+            'domains': self._svc_name(),
+            'maintenance': app_settings.maintenance,
+            'routable': app_settings.routable,
+            'proxyDomain': self.app.id,
+            'proxyLocations': self.path_pattern
+        }
+
+    def _update_service(self, namespace, app_type, routable, annotations):  # noqa
+        """Update application service with all the various required information"""
+        svc_name = "{}-{}".format(namespace, app_type)
+        service = self._fetch_service_config(namespace, svc_name)
+        old_service = service.copy()  # in case anything fails for rollback
+
+        try:
+            # Update service information
+            for key, value in annotations.items():
+                if value is not None:
+                    service['metadata']['annotations']['router.deis.io/%s' % key] = str(value)
+                else:
+                    service['metadata']['annotations'].pop('router.deis.io/%s' % key, None)
+            if routable:
+                service['metadata']['labels']['router.deis.io/routable'] = 'true'
+            else:
+                # delete the annotation
+                service['metadata']['labels'].pop('router.deis.io/routable', None)
+
+            # Set app type selector
+            service['spec']['selector']['type'] = app_type
+
+            self._scheduler.svc.update(namespace, svc_name, data=service)
+        except Exception as e:
+            # Fix service to old port and app type
+            self._scheduler.svc.update(namespace, svc_name, data=old_service)
+            raise ServiceUnavailable(str(e)) from e

rootfs/api/serializers.py

@@ -472,6 +472,40 @@ def ToACE(x): return idna.alabel(x).decode("utf-8", "strict")
         return aceValue
 
 
+class ServiceSerializer(serializers.ModelSerializer):
+    """Serialize a :class:`~api.models.Service` model."""
+
+    app = serializers.SlugRelatedField(slug_field='id', queryset=models.App.objects.all())
+    owner = serializers.ReadOnlyField(source='owner.username')
+    procfile_type = serializers.CharField(allow_blank=False, allow_null=False, required=True)
+    path_pattern = serializers.CharField(allow_blank=False, allow_null=False, required=True)
+
+    class Meta:
+        """Metadata options for a :class:`ServiceSerializer`."""
+        model = models.Service
+        fields = ['owner', 'created', 'updated', 'app', 'procfile_type', 'path_pattern']
+        read_only_fields = ['uuid']
+
+    def validate_procfile_type(self, value):
+        if not re.match(PROCTYPE_MATCH, value):
+            raise serializers.ValidationError(PROCTYPE_MISMATCH_MSG)
+
+        return value
+
+    def validate_path_pattern(self, value):
+        for pattern in str(value).split(","):
+            if not pattern.strip():
+                raise serializers.ValidationError(
+                    "Service value should be valid regex (or set of regex split by comma)")
+            try:
+                re.compile(pattern)
+            except re.error as e:
+                raise serializers.ValidationError(
+                    "Service value should be valid regex (or set of regex split by comma)")
+
+        return value
+
+
 class CertificateSerializer(serializers.ModelSerializer):
     """Serialize a :class:`~api.models.Cert` model."""