ref(models): split up model classes into their own files

Has the __init__ file still source in all the classes as needed to minimise changes in tests and migrations

Author: helgi

rootfs/api/models/__init__.py

@@ -0,0 +1,58 @@
+from django.conf import settings
+from django.db import models
+from jsonfield import JSONField
+
+from api.models import UuidAuditedModel
+
+
+class Build(UuidAuditedModel):
+    """
+    Instance of a software build used by runtime nodes
+    """
+
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
+    app = models.ForeignKey('App')
+    image = models.CharField(max_length=256)
+
+    # optional fields populated by builder
+    sha = models.CharField(max_length=40, blank=True)
+    procfile = JSONField(default={}, blank=True)
+    dockerfile = models.TextField(blank=True)
+
+    class Meta:
+        get_latest_by = 'created'
+        ordering = ['-created']
+        unique_together = (('app', 'uuid'),)
+
+    def create(self, user, *args, **kwargs):
+        latest_release = self.app.release_set.latest()
+        source_version = 'latest'
+        if self.sha:
+            source_version = 'git-{}'.format(self.sha)
+        new_release = latest_release.new(user,
+                                         build=self,
+                                         config=latest_release.config,
+                                         source_version=source_version)
+        try:
+            self.app.deploy(user, new_release)
+            return new_release
+        except RuntimeError:
+            if 'new_release' in locals():
+                new_release.delete()
+            raise
+
+    def save(self, **kwargs):
+        try:
+            previous_build = self.app.build_set.latest()
+            to_destroy = []
+            for proctype in previous_build.procfile:
+                if proctype not in self.procfile:
+                    for c in self.app.container_set.filter(type=proctype):
+                        to_destroy.append(c)
+            self.app._destroy_containers(to_destroy)
+        except Build.DoesNotExist:
+            pass
+        return super(Build, self).save(**kwargs)
+
+    def __str__(self):
+        return "{0}-{1}".format(self.app.id, str(self.uuid)[:7])

rootfs/api/models/app.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+
+from django.db import models
+from django.conf import settings
+from django.core.exceptions import ValidationError, SuspiciousOperation
+
+from OpenSSL import crypto
+from api.models import AuditedModel
+
+
+def validate_certificate(value):
+    try:
+        crypto.load_certificate(crypto.FILETYPE_PEM, value)
+    except crypto.Error as e:
+        raise ValidationError('Could not load certificate: {}'.format(e))
+
+
+class Certificate(AuditedModel):
+    """
+    Public and private key pair used to secure application traffic at the router.
+    """
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
+    # there is no upper limit on the size of an x.509 certificate
+    certificate = models.TextField(validators=[validate_certificate])
+    key = models.TextField()
+    # X.509 certificates allow any string of information as the common name.
+    common_name = models.TextField(unique=True)
+    expires = models.DateTimeField()
+
+    def __str__(self):
+        return self.common_name
+
+    def _get_certificate(self):
+        try:
+            return crypto.load_certificate(crypto.FILETYPE_PEM, self.certificate)
+        except crypto.Error as e:
+            raise SuspiciousOperation(e)
+
+    def save(self, *args, **kwargs):
+        certificate = self._get_certificate()
+        if not self.common_name:
+            self.common_name = certificate.get_subject().CN
+
+        if not self.expires:
+            # https://pyopenssl.readthedocs.org/en/latest/api/crypto.html#OpenSSL.crypto.X509.get_notAfter
+            # Convert bytes to string
+            timestamp = certificate.get_notAfter().decode(encoding='UTF-8')
+            # convert openssl's expiry date format to Django's DateTimeField format
+            self.expires = datetime.strptime(timestamp, '%Y%m%d%H%M%SZ')
+
+        return super(Certificate, self).save(*args, **kwargs)

rootfs/api/models/build.py

@@ -0,0 +1,74 @@
+from django.conf import settings
+from django.db import models
+from jsonfield import JSONField
+
+from api.models import UuidAuditedModel
+
+
+class Config(UuidAuditedModel):
+    """
+    Set of configuration values applied as environment variables
+    during runtime execution of the Application.
+    """
+
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
+    app = models.ForeignKey('App')
+    values = JSONField(default={}, blank=True)
+    memory = JSONField(default={}, blank=True)
+    cpu = JSONField(default={}, blank=True)
+    tags = JSONField(default={}, blank=True)
+
+    class Meta:
+        get_latest_by = 'created'
+        ordering = ['-created']
+        unique_together = (('app', 'uuid'),)
+
+    def __str__(self):
+        return "{}-{}".format(self.app.id, str(self.uuid)[:7])
+
+    def healthcheck(self):
+        # Update healthcheck - Scheduler determines the app type
+        path = self.values.get('HEALTHCHECK_URL', '/')
+        timeout = int(self.values.get('HEALTHCHECK_TIMEOUT', 1))
+        delay = int(self.values.get('HEALTHCHECK_INITIAL_DELAY', 10))
+        port = int(self.values.get('HEALTHCHECK_PORT', 8080))
+
+        return {'path': path, 'timeout': timeout, 'delay': delay, 'port': port}
+
+    def save(self, **kwargs):
+        """merge the old config with the new"""
+        try:
+            previous_config = self.app.config_set.latest()
+            for attr in ['cpu', 'memory', 'tags', 'values']:
+                # Guard against migrations from older apps without fixes to
+                # JSONField encoding.
+                try:
+                    data = getattr(previous_config, attr).copy()
+                except AttributeError:
+                    data = {}
+
+                try:
+                    new_data = getattr(self, attr).copy()
+                except AttributeError:
+                    new_data = {}
+
+                data.update(new_data)
+                # remove config keys if we provided a null value
+                [data.pop(k) for k, v in new_data.items() if v is None]
+                setattr(self, attr, data)
+        except Config.DoesNotExist:
+            pass
+
+        # verify the tags exist on any nodes as labels
+        if self.tags:
+            # Get all nodes with label selectors
+            nodes = self._scheduler._get_nodes(labels=self.tags).json()
+            if not nodes['items']:
+                labels = ['{}={}'.format(key, value) for key, value in self.tags.items()]
+                raise EnvironmentError(
+                    'These tags do not match labels on kubernetes nodes: {}'.format(
+                        ', '.join(labels)
+                    )
+                )
+
+        return super(Config, self).save(**kwargs)

rootfs/api/models/certificate.py

@@ -0,0 +1,153 @@
+import logging
+
+from django.conf import settings
+from django.db import models
+
+from api.models import UuidAuditedModel, log_event
+
+logger = logging.getLogger(__name__)
+
+
+def close_db_connections(func, *args, **kwargs):
+    """
+    Decorator to explicitly close db connections during threaded execution
+
+    Note this is necessary to work around:
+    https://code.djangoproject.com/ticket/22420
+    """
+    def _close_db_connections(*args, **kwargs):
+        ret = None
+        try:
+            ret = func(*args, **kwargs)
+        finally:
+            from django.db import connections
+            for conn in connections.all():
+                conn.close()
+        return ret
+    return _close_db_connections
+
+
+class Container(UuidAuditedModel):
+    """
+    Docker container used to securely host an application process.
+    """
+
+    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
+    app = models.ForeignKey('App')
+    release = models.ForeignKey('Release')
+    type = models.CharField(max_length=128, blank=False)
+    num = models.PositiveIntegerField()
+
+    @property
+    def state(self):
+        return self._scheduler.state(self.job_id).name
+
+    def short_name(self):
+        return "{}.{}.{}".format(self.app.id, self.type, self.num)
+    short_name.short_description = 'Name'
+
+    def __str__(self):
+        return self.short_name()
+
+    class Meta:
+        get_latest_by = '-created'
+        ordering = ['created']
+
+    @property
+    def job_id(self):
+        version = "v{}".format(self.release.version)
+        return "{self.app.id}_{version}.{self.type}.{self.num}".format(**locals())
+
+    def _get_command(self):
+        try:
+            # if this is not procfile-based app, ensure they cannot break out
+            # and run arbitrary commands on the host
+            # FIXME: remove slugrunner's hardcoded entrypoint
+            if self.release.build.dockerfile or not self.release.build.sha:
+                return "bash -c '{}'".format(self.release.build.procfile[self.type])
+            else:
+                return 'start {}'.format(self.type)
+        # if the key is not present or if a parent attribute is None
+        except (KeyError, TypeError, AttributeError):
+            # handle special case for Dockerfile deployments
+            return '' if self.type == 'cmd' else 'start {}'.format(self.type)
+
+    _command = property(_get_command)
+
+    def clone(self, release):
+        c = Container.objects.create(owner=self.owner,
+                                     app=self.app,
+                                     release=release,
+                                     type=self.type,
+                                     num=self.num)
+        return c
+
+    @close_db_connections
+    def create(self):
+        image = self.release.image
+        kwargs = {'memory': self.release.config.memory,
+                  'cpu': self.release.config.cpu,
+                  'tags': self.release.config.tags,
+                  'envs': self.release.config.values}
+        try:
+            self._scheduler.create(
+                name=self.job_id,
+                image=image,
+                command=self._command,
+                **kwargs
+            )
+        except Exception as e:
+            err = '{} (create): {}'.format(self.job_id, e)
+            log_event(self.app, err, logging.ERROR)
+            raise
+
+    @close_db_connections
+    def start(self):
+        try:
+            self._scheduler.start(self.job_id)
+        except Exception as e:
+            err = '{} (start): {}'.format(self.job_id, e)
+            log_event(self.app, err, logging.WARNING)
+            raise
+
+    @close_db_connections
+    def stop(self):
+        try:
+            self._scheduler.stop(self.job_id)
+        except Exception as e:
+            err = '{} (stop): {}'.format(self.job_id, e)
+            log_event(self.app, err, logging.ERROR)
+            raise
+
+    @close_db_connections
+    def destroy(self):
+        try:
+            self._scheduler.destroy(self.job_id)
+        except Exception as e:
+            err = '{} (destroy): {}'.format(self.job_id, e)
+            log_event(self.app, err, logging.ERROR)
+            raise
+
+    def run(self, command):
+        """Run a one-off command"""
+        if self.release.build is None:
+            raise EnvironmentError('No build associated with this release '
+                                   'to run this command')
+        image = self.release.image
+        entrypoint = '/bin/bash'
+        # if this is a procfile-based app, switch the entrypoint to slugrunner's default
+        # FIXME: remove slugrunner's hardcoded entrypoint
+        if self.release.build.procfile and \
+           self.release.build.sha and not \
+           self.release.build.dockerfile:
+            entrypoint = '/runner/init'
+            command = "'{}'".format(command)
+        else:
+            command = "-c '{}'".format(command)
+        try:
+            rc, output = self._scheduler.run(self.job_id, image, entrypoint, command)
+            return rc, output
+        except Exception as e:
+            err = '{} (run): {}'.format(self.job_id, e)
+            log_event(self.app, err, logging.ERROR)
+            raise