models.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
Data models for the Deis API.
"""
# pylint: disable=R0903,W0232

from __future__ import unicode_literals
import os
import subprocess

from celery.canvas import group
from django.conf import settings
from django.contrib.auth.models import User
from django.db import models
from django.db.models.signals import post_save
from django.dispatch import receiver
from django.dispatch.dispatcher import Signal
from django.utils.encoding import python_2_unicode_compatible

from api import fields
from provider import import_provider_module


# define custom signals
release_signal = Signal(providing_args=['user', 'app'])

# define custom exceptions


class ScalingError(Exception):
    pass

# base models


class AuditedModel(models.Model):
    """Add created and updated fields to a model."""

    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)

    class Meta:
        """Mark :class:`AuditedModel` as abstract."""
        abstract = True


class UuidAuditedModel(AuditedModel):
    """Add a UUID primary key to an :class:`AuditedModel`."""

    uuid = fields.UuidField('UUID', primary_key=True)

    class Meta:
        """Mark :class:`UuidAuditedModel` as abstract."""
        abstract = True

# deis core models


@python_2_unicode_compatible
class Key(UuidAuditedModel):
    """An SSH public key."""

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.CharField(max_length=128)
    public = models.TextField(unique=True)

    class Meta:
        verbose_name = 'SSH Key'
        unique_together = (('owner', 'id'))

    def __str__(self):
        return "{}...{}".format(self.public[:18], self.public[-31:])

    def save(self, *args, **kwargs):
        super(Key, self).save(*args, **kwargs)
        self.owner.publish()

    def delete(self, *args, **kwargs):
        super(Key, self).delete(*args, **kwargs)
        self.owner.publish()


@python_2_unicode_compatible
class ProviderManager(models.Manager):
    """Manage database interactions for :class:`Provider`."""

    def seed(self, user, **kwargs):
        """
        Seeds the database with Providers for clouds supported by Deis.
        """
        providers = (('ec2', 'ec2'), ('mock', 'mock'))
        for p_id, p_type in providers:
            self.create(owner=user, id=p_id, type=p_type, creds='{}')


@python_2_unicode_compatible
class Provider(UuidAuditedModel):
    """Cloud provider settings for a user.

    Available as `user.provider_set`.
    """

    objects = ProviderManager()

    PROVIDERS = (
        ('ec2', 'Amazon Elastic Compute Cloud (EC2)'),
        ('mock', 'Mock Reference Provider'),
    )

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.SlugField(max_length=64)
    type = models.SlugField(max_length=16, choices=PROVIDERS)
    creds = fields.CredentialsField(blank=True)

    class Meta:
        unique_together = (('owner', 'id'),)

    def __str__(self):
        return "{}-{}".format(self.id, self.get_type_display())

    def flat(self):
        return {'id': self.id,
                'type': self.type,
                'creds': dict(self.creds)}


@python_2_unicode_compatible
class FlavorManager(models.Manager):
    """Manage database interactions for :class:`Flavor`."""

    def seed(self, user, **kwargs):
        """Seed the database with default Flavors for each cloud region."""
        for provider_type in ('mock', 'ec2'):
            provider = import_provider_module(provider_type)
            flavors = provider.seed_flavors()
            p = Provider.objects.get(owner=user, id=provider_type)
            for flavor in flavors:
                flavor['provider'] = p
                Flavor.objects.create(owner=user, **flavor)


@python_2_unicode_compatible
class Flavor(UuidAuditedModel):
    """
    Virtual machine flavors associated with a Provider

    Params is a JSON field including unstructured data
    for provider API calls, like region, zone, and size.
    """
    objects = FlavorManager()

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.SlugField(max_length=64)
    provider = models.ForeignKey('Provider')
    params = fields.ParamsField(blank=True)

    class Meta:
        unique_together = (('owner', 'id'),)

    def __str__(self):
        return self.id

    def flat(self):
        return {'id': self.id,
                'creds': dict(self.provider.creds),
                'provider': self.provider.id,
                'params': self.params}


@python_2_unicode_compatible
class FormationManager(models.Manager):
    """Manage database interactions for :class:`Formation`."""

    def next_container_node(self, formation, container_type, reverse=False):
        count = []
        layers = formation.layer_set.filter(runtime=True)
        runtime_nodes = []
        for l in layers:
            runtime_nodes.extend(Node.objects.filter(
                formation=formation, layer=l).order_by('created'))
        container_map = {n: [] for n in runtime_nodes}
        containers = list(Container.objects.filter(
            formation=formation, type=container_type).order_by('created'))
        for c in containers:
            container_map[c.node].append(c)
        for n in container_map.keys():
            # (2, node3), (2, node2), (3, node1)
            count.append((len(container_map[n]), n))
        if not count:
            raise ScalingError('No nodes available for containers')
        count.sort()
        # reverse means order by greatest # of containers, otherwise fewest
        if reverse:
            count.reverse()
        return count[0][1]


@python_2_unicode_compatible
class Formation(UuidAuditedModel):

    """
    Formation of nodes used to host applications
    """
    objects = FormationManager()

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.SlugField(max_length=64, unique=True)
    domain = models.CharField(max_length=128, blank=True, null=True)
    nodes = fields.JSONField(default='{}', blank=True)

    class Meta:
        unique_together = (('owner', 'id'),)

    def __str__(self):
        return self.id

    def flat(self):
        return {'id': self.id,
                'domain': self.domain,
                'nodes': self.nodes}

    def build(self):
        tasks.build_formation.delay(self).wait()

    def destroy(self, *args, **kwargs):
        tasks.destroy_formation.delay(self).wait()

    def publish(self):
        data = self.calculate()
        CM.publish_formation(self.flat(), data)
        return data

    def converge(self, **kwargs):
        databag = self.publish()
        tasks.converge_formation.delay(self).wait()
        return databag

    def calculate(self):
        """Return a representation of this formation for config management"""
        d = {}
        d['id'] = self.id
        d['domain'] = self.domain
        d['nodes'] = {}
        proxies = []
        for n in self.node_set.all():
            d['nodes'][n.id] = {'fqdn': n.fqdn,
                                'runtime': n.layer.runtime,
                                'proxy': n.layer.proxy}
            if n.layer.proxy is True:
                proxies.append(n.fqdn)
        d['apps'] = {}
        for a in self.app_set.all():
            d['apps'][a.id] = a.calculate()
            d['apps'][a.id]['proxy'] = {}
            d['apps'][a.id]['proxy']['nodes'] = proxies
            d['apps'][a.id]['proxy']['algorithm'] = 'round_robin'
            d['apps'][a.id]['proxy']['port'] = 80
            d['apps'][a.id]['proxy']['backends'] = []
            d['apps'][a.id]['containers'] = containers = {}
            for c in self.container_set.all().order_by('created'):
                port = 5000 + c.num
                containers.setdefault(c.type, {})
                containers[c.type].update(
                    {c.num: "{0}:{1}".format(c.node.id, port)})
                if c.type == 'web':
                    d['apps'][a.id]['proxy']['backends'].append(
                        "{0}:{1}".format(c.node.fqdn, port))
        return d


@python_2_unicode_compatible
class Layer(UuidAuditedModel):

    """
    Layer of nodes used by the formation

    All nodes in a layer share the same flavor and configuration.

    The layer stores SSH settings used to trigger node convergence,
    as well as other configuration used during node bootstrapping
    (e.g. Chef Run List, Chef Environment)
    """

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.SlugField(max_length=64)

    formation = models.ForeignKey('Formation')
    flavor = models.ForeignKey('Flavor')

    proxy = models.BooleanField(default=False)
    runtime = models.BooleanField(default=False)

    ssh_username = models.CharField(max_length=64, default='ubuntu')
    ssh_private_key = models.TextField()
    ssh_public_key = models.TextField()
    ssh_port = models.SmallIntegerField(default=22)

    # example: {'run_list': [deis::runtime'], 'environment': 'dev'}
    config = fields.JSONField(default='{}', blank=True)

    class Meta:
        unique_together = (('formation', 'id'),)

    def __str__(self):
        return self.id

    def flat(self):
        return {'id': self.id,
                'provider_type': self.flavor.provider.type,
                'creds': dict(self.flavor.provider.creds),
                'formation': self.formation.id,
                'flavor': self.flavor.id,
                'params': dict(self.flavor.params),
                'proxy': self.proxy,
                'runtime': self.runtime,
                'ssh_username': self.ssh_username,
                'ssh_private_key': self.ssh_private_key,
                'ssh_public_key': self.ssh_public_key,
                'ssh_port': self.ssh_port,
                'config': dict(self.config)}

    def build(self):
        return tasks.build_layer.delay(self).wait()

    def destroy(self):
        return tasks.destroy_layer.delay(self).wait()


@python_2_unicode_compatible
class NodeManager(models.Manager):

    def new(self, formation, layer):
        existing_nodes = self.filter(formation=formation, layer=layer).order_by('-created')
        if existing_nodes:
            next_num = existing_nodes[0].num + 1
        else:
            next_num = 1
        node = self.create(owner=formation.owner,
                           formation=formation,
                           layer=layer,
                           num=next_num,
                           id="{0}-{1}-{2}".format(formation.id, layer.id, next_num))
        return node

    def scale(self, formation, structure, **kwargs):
        """Scale layers up or down to match requested structure."""
        funcs = []
        changed = False
        for layer_id, requested in structure.items():
            layer = formation.layer_set.get(id=layer_id)
            nodes = list(layer.node_set.all().order_by('created'))
            diff = requested - len(nodes)
            if diff == 0:
                continue
            while diff < 0:
                node = nodes.pop(0)
                funcs.append(tasks.destroy_node.si(node))
                diff = requested - len(nodes)
                changed = True
            while diff > 0:
                node = self.new(formation, layer)
                nodes.append(node)
                funcs.append(tasks.build_node.si(node))
                diff = requested - len(nodes)
                changed = True
        # launch/terminate nodes in parallel
        if funcs:
            group(*funcs).apply_async().join()
        # always scale and balance every application
        if nodes:
            for app in formation.app_set.all():
                Container.objects.scale(app, app.containers)
                Container.objects.balance(formation)
        # save new structure now that scaling was successful
        formation.nodes.update(structure)
        formation.save()
        # force-converge nodes if there were new nodes or container rebalancing
        if changed:
            return formation.converge()
        return formation.calculate()


@python_2_unicode_compatible
class Node(UuidAuditedModel):
    """
    Node used to host containers

    List of nodes available as `formation.nodes`
    """

    objects = NodeManager()

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.CharField(max_length=64)
    formation = models.ForeignKey('Formation')
    layer = models.ForeignKey('Layer')
    num = models.PositiveIntegerField()

    # TODO: add celery beat tasks for monitoring node health
    status = models.CharField(max_length=64, default='up')

    provider_id = models.SlugField(max_length=64, blank=True, null=True)
    fqdn = models.CharField(max_length=256, blank=True, null=True)
    status = fields.NodeStatusField(blank=True, null=True)

    class Meta:
        unique_together = (('formation', 'id'),)

    def __str__(self):
        return self.id

    def flat(self):
        return {'id': self.id,
                'provider_type': self.layer.flavor.provider.type,
                'formation': self.formation.id,
                'layer': self.layer.id,
                'creds': dict(self.layer.flavor.provider.creds),
                'params': dict(self.layer.flavor.params),
                'runtime': self.layer.runtime,
                'proxy': self.layer.proxy,
                'ssh_username': self.layer.ssh_username,
                'ssh_private_key': self.layer.ssh_private_key,
                'config': dict(self.layer.config),
                'provider_id': self.provider_id,
                'fqdn': self.fqdn}

    def build(self):
        return tasks.build_node.delay(self).wait()

    def destroy(self):
        return tasks.destroy_node.delay(self).wait()

    def converge(self):
        return tasks.converge_node.delay(self).wait()

    def run(self, command, **kwargs):
        return tasks.run_node.delay(self, command).wait()


@python_2_unicode_compatible
class App(UuidAuditedModel):
    """
    Application used to service requests on behalf of end-users
    """

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    id = models.SlugField(max_length=64, unique=True)
    formation = models.ForeignKey('Formation')

    containers = fields.JSONField(default='{}', blank=True)

    def __str__(self):
        return self.id

    def flat(self):
        return {'id': self.id,
                'formation': self.formation.id,
                'containers': dict(self.containers)}

    def build(self):
        config = Config.objects.create(
            version=1, owner=self.owner, app=self, values={})
        Release.objects.create(
            version=1, owner=self.owner, app=self, config=config)
        self.formation.publish()
        tasks.build_app.delay(self).wait()

    def destroy(self):
        tasks.destroy_app.delay(self).wait()

    def publish(self):
        """Publish the application to configuration management"""
        data = self.calculate()
        CM.publish_app(self.flat(), data)
        return data

    def converge(self):
        self.publish()
        self.formation.converge()

    def calculate(self):
        """Return a representation for configuration management"""
        d = {}
        d['id'] = self.id
        d['release'] = {}
        releases = self.release_set.all().order_by('-created')
        if releases:
            release = releases[0]
            d['release']['version'] = release.version
            d['release']['config'] = release.config.values
            d['release']['image'] = release.image
            d['release']['build'] = {}
            if release.build:
                d['release']['build']['url'] = release.build.url
                d['release']['build']['procfile'] = release.build.procfile
        d['proxies'] = []
        for n in self.formation.node_set.filter(layer__proxy=True):
            d['proxies'].append(n.fqdn)
        # TODO: add proper sharing and access controls
        d['users'] = {}
        for u in (self.owner.username,):
            d['users'][u] = 'admin'
        return d

    def logs(self):
        """Return aggregated log data for this application."""
        path = os.path.join(settings.DEIS_LOG_DIR, self.id + '.log')
        if not os.path.exists(path):
            raise EnvironmentError('Could not locate logs')
        data = subprocess.check_output(['tail', '-n', str(settings.LOG_LINES), path])
        return data

    def run(self, command):
        """Run a one-off command in an ephemeral app container."""
        nodes = self.formation.node_set.order_by('?')
        releases = self.release_set.order_by('-created')
        if not nodes:
            raise EnvironmentError('No nodes available to run command')
        if not releases:
            raise EnvironmentError('No release available to run command')
        node, release = nodes[0], releases[0]
        # prepare ssh command
        version = release.version
        docker_args = ' '.join(
            ['-v',
             '/opt/deis/runtime/slugs/{app_id}-{version}/app:/app'.format(**locals()),
             release.image])
        base_cmd = "export HOME=/app; cd /app && for profile in " \
                   "`find /app/.profile.d/*.sh -type f`; do . $profile; done"
        command = "/bin/sh -c '{base_cmd} && {command}'".format(**locals())
        command = "sudo docker run {docker_args} {command}".format(**locals())
        return node.run(self, command)


@python_2_unicode_compatible
class ContainerManager(models.Manager):

    def scale(self, app, structure, **kwargs):
        """Scale containers up or down to match requested."""
        requested_containers = structure.copy()
        formation = app.formation
        # increment new container nums off the most recent container
        all_containers = app.container_set.all().order_by('-created')
        container_num = 1 if not all_containers else all_containers[0].num + 1
        # iterate and scale by container type (web, worker, etc)
        changed = False
        for container_type in requested_containers.keys():
            containers = list(app.container_set.filter(type=container_type).order_by('created'))
            requested = requested_containers.pop(container_type)
            diff = requested - len(containers)
            if diff == 0:
                continue
            changed = True
            while diff < 0:
                # get the next node with the most containers
                node = Formation.objects.next_container_node(
                    formation, container_type, reverse=True)
                # delete a container attached to that node
                for c in containers:
                    if node == c.node:
                        containers.remove(c)
                        c.delete()
                        diff += 1
                        break
            while diff > 0:
                # get the next node with the fewest containers
                node = Formation.objects.next_container_node(formation, container_type)
                c = Container.objects.create(owner=app.owner,
                                             formation=formation,
                                             node=node,
                                             app=app,
                                             type=container_type,
                                             num=container_num)
                containers.append(c)
                container_num += 1
                diff -= 1
        return changed

    def balance(self, formation, **kwargs):
        runtime_nodes = formation.node_set.filter(layer__runtime=True).order_by('created')
        all_containers = self.filter(formation=formation).order_by('-created')
        # get the next container number (e.g. web.19)
        container_num = 1 if not all_containers else all_containers[0].num + 1
        changed = False
        # iterate by unique container type
        for container_type in set([c.type for c in all_containers]):
            # map node container counts => { 2: [b3, b4], 3: [ b1, b2 ] }
            n_map = {}
            for node in runtime_nodes:
                ct = len(node.container_set.filter(type=container_type))
                n_map.setdefault(ct, []).append(node)
            # loop until diff between min and max is 1 or 0
            while max(n_map.keys()) - min(n_map.keys()) > 1:
                # get the most over-utilized node
                n_max = max(n_map.keys())
                n_over = n_map[n_max].pop(0)
                if len(n_map[n_max]) == 0:
                    del n_map[n_max]
                # get the most under-utilized node
                n_min = min(n_map.keys())
                n_under = n_map[n_min].pop(0)
                if len(n_map[n_min]) == 0:
                    del n_map[n_min]
                # delete the oldest container from the most over-utilized node
                c = n_over.container_set.filter(type=container_type).order_by('created')[0]
                app = c.app  # pull ref to app for recreating the container
                c.delete()
                # create a container on the most under-utilized node
                self.create(owner=formation.owner,
                            formation=formation,
                            app=app,
                            type=container_type,
                            num=container_num,
                            node=n_under)
                container_num += 1
                # update the n_map accordingly
                for n in (n_over, n_under):
                    ct = len(n.container_set.filter(type=container_type))
                    n_map.setdefault(ct, []).append(n)
                changed = True
        return changed


@python_2_unicode_compatible
class Container(UuidAuditedModel):
    """
    Docker container used to securely host an application process.
    """

    objects = ContainerManager()

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    formation = models.ForeignKey('Formation')
    node = models.ForeignKey('Node')
    app = models.ForeignKey('App')
    type = models.CharField(max_length=128)
    num = models.PositiveIntegerField()

    # TODO: add celery beat tasks for monitoring node health
    status = models.CharField(max_length=64, default='up')

    def short_name(self):
        return "{}.{}".format(self.type, self.num)
    short_name.short_description = 'Name'

    def __str__(self):
        return "{0} {1}".format(self.formation.id, self.short_name())

    class Meta:
        get_latest_by = '-created'
        ordering = ['created']
        unique_together = (('app', 'type', 'num'),)


@python_2_unicode_compatible
class Config(UuidAuditedModel):
    """
    Set of configuration values applied as environment variables
    during runtime execution of the Application.
    """

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    app = models.ForeignKey('App')
    version = models.PositiveIntegerField()

    values = fields.EnvVarsField(default='{}', blank=True)

    class Meta:
        get_latest_by = 'created'
        ordering = ['-created']
        unique_together = (('app', 'version'),)

    def __str__(self):
        return "{0}-v{1}".format(self.app.id, self.version)


@python_2_unicode_compatible
class Build(UuidAuditedModel):
    """
    Instance of a software build used by runtime nodes
    """

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    app = models.ForeignKey('App')
    sha = models.CharField('SHA', max_length=255, blank=True)
    output = models.TextField(blank=True)

    procfile = fields.ProcfileField(blank=True)
    dockerfile = models.TextField(blank=True)
    config = fields.EnvVarsField(blank=True)

    url = models.URLField('URL')
    size = models.IntegerField(blank=True, null=True)
    checksum = models.CharField(max_length=255, blank=True)

    class Meta:
        get_latest_by = 'created'
        ordering = ['-created']
        unique_together = (('app', 'uuid'),)

    def __str__(self):
        return "{0}-{1}".format(self.app.id, self.sha)

    @classmethod
    def push(cls, push):
        """Process a push from a local Git server.

        Creates a new Build and returns the application's
        databag for processing by the git-receive hook
        """
        # SECURITY:
        # we assume the first part of the ssh key name
        # is the authenticated user because we trust gitosis
        username = push.pop('username').split('_')[0]
        # retrieve the user and app instances
        user = User.objects.get(username=username)
        app = App.objects.get(owner=user, id=push.pop('app'))
        # merge the push with the required model instances
        push['owner'] = user
        push['app'] = app
        # create the build
        new_build = cls.objects.create(**push)
        # send a release signal
        release_signal.send(sender=push, build=new_build, app=app, user=user)
        # see if we need to scale an initial web container
        if len(app.formation.node_set.filter(layer__runtime=True)) > 0 and \
           len(app.formation.container_set.filter(type='web')) < 1:
            # scale an initial web containers
            Container.objects.scale(app, {'web': 1})
        # publish the app, triggering a formation converge
        return app.publish()


@python_2_unicode_compatible
class Release(UuidAuditedModel):
    """
    Software release deployed by the application platform

    Releases contain a Build and a Config.
    """

    owner = models.ForeignKey(settings.AUTH_USER_MODEL)
    app = models.ForeignKey('App')
    version = models.PositiveIntegerField()

    config = models.ForeignKey('Config')
    image = models.CharField(max_length=256, default='deis/buildstep')
    build = models.ForeignKey('Build', blank=True, null=True)

    class Meta:
        get_latest_by = 'created'
        ordering = ['-created']
        unique_together = (('app', 'version'),)

    def __str__(self):
        return "{0}-v{1}".format(self.app.id, self.version)

    def rollback(self):
        # create a rollback log entry
        # call run
        raise NotImplementedError


@receiver(release_signal)
def new_release(sender, **kwargs):
    """
    Catch a release_signal and create a new release
    using the latest Build and Config for an application.

    Releases start at v1 and auto-increment.
    """
    user, app, = kwargs['user'], kwargs['app']
    last_release = Release.objects.filter(app=app).order_by('-created')[0]
    image = kwargs.get('image', last_release.image)
    config = kwargs.get('config', last_release.config)
    build = kwargs.get('build', last_release.build)
    # overwrite config with build.config if the keys don't exist
    if build and build.config:
        new_values = {}
        for k, v in build.config.items():
            if not k in config.values:
                new_values[k] = v
        if new_values:
            # update with current config
            new_values.update(config.values)
            config = Config.objects.create(
                version=config.version + 1, owner=user,
                app=app, values=new_values)
    # create new release and auto-increment version
    new_version = last_release.version + 1
    release = Release.objects.create(
        owner=user, app=app, image=image, config=config,
        build=build, version=new_version)
    # converge the application
    app.converge()
    return release


def _user_flat(self):
    return {'username': self.username}


def _user_calculate(self):
    data = {'id': self.username, 'ssh_keys': {}}
    for k in self.key_set.all():
        data['ssh_keys'][k.id] = k.public
    return data


def _user_publish(self):
    CM.publish_user(self.flat(), self.calculate())


# attach to built-in django user
User.flat = _user_flat
User.calculate = _user_calculate
User.publish = _user_publish

# define update/delete callbacks for synchronizing
# models with the configuration management backend


def _publish_to_cm(**kwargs):
    kwargs['instance'].publish()


def _publish_user_to_cm(**kwargs):
    if kwargs.get('update_fields') == frozenset(['last_login']):
        return
    kwargs['instance'].publish()

# use django signals to synchronize database updates with
# the configuration management backend
post_save.connect(_publish_to_cm, sender=App, dispatch_uid='api.models')
post_save.connect(_publish_to_cm, sender=Formation, dispatch_uid='api.models')
post_save.connect(_publish_user_to_cm, sender=User, dispatch_uid='api.models')


# now that we've defined models that may be imported by celery tasks
# import tasks and user-defined config management module
from api import tasks

import importlib
CM = importlib.import_module(settings.CM_MODULE)