Merge branch 'master' into tutorial-docs

Author: mboersma

.travis.yml

@@ -16,7 +16,7 @@ services:
   - rabbitmq
 
 before_install:
-  - sudo apt-get install -qq libevent-dev libpq-dev
+  - sudo apt-get install -qq libpq-dev
 
 install:
   - pip install -r dev_requirements.txt coveralls --use-mirrors

Berksfile

@@ -1,4 +1,4 @@
-source "http://api.berkshelf.com"
+site :opscode
 
 cookbook 'apt'
 cookbook 'rsyslog'

Berksfile.lock

@@ -1,5 +1,5 @@
 {
-  "dependencies": {
+  "sources": {
     "apt": {
       "locked_version": "2.3.0"
     },
@@ -10,9 +10,9 @@
       "locked_version": "2.2.2"
     },
     "deis": {
-      "locked_version": "0.2.1",
+      "locked_version": "0.3.0",
       "git": "https://github.com/opdemand/deis-cookbook.git",
-      "ref": "a2698641d2042a3626df5874763ab6535ee2892b"
+      "ref": "cb7ea38a10be3ac9f80a412f81cdc23ce0337c0e"
     }
   }
 }

Gemfile

@@ -2,7 +2,8 @@ source 'https://rubygems.org'
 
 gem 'chef'
 gem 'foodcritic'
-gem 'berkshelf', '3.0.0.beta3' # needed for --ssl-verify=false support
+#gem 'berkshelf', '3.0.0.beta3' # needed for --ssl-verify=false support
+gem "berkshelf", "~> 2.0.10"
 
 gem 'knife-ec2'
 gem 'knife-rackspace'

Gemfile.lock

@@ -1,36 +1,40 @@
 GEM
   remote: https://rubygems.org/
   specs:
+    activesupport (3.2.16)
+      i18n (~> 0.6, >= 0.6.4)
+      multi_json (~> 1.0)
     addressable (2.3.5)
     akami (1.2.0)
       gyoku (>= 0.4.0)
       nokogiri (>= 1.4.0)
-    berkshelf (3.0.0.beta3)
+    berkshelf (2.0.10)
+      activesupport (~> 3.2.0)
       addressable (~> 2.3.4)
-      buff-config (~> 0.2)
-      buff-extensions (~> 0.4)
       buff-shell_out (~> 0.1)
+      chozo (>= 0.6.1)
       faraday (>= 0.8.5)
       hashie (>= 2.0.2)
       minitar (~> 0.5.4)
+      rbzip2 (~> 0.2.0)
       retryable (~> 1.3.3)
-      ridley (~> 1.6)
-      solve (>= 0.8.0)
+      ridley (~> 1.5.0)
+      solve (>= 0.5.0)
       thor (~> 0.18.0)
     buff-config (0.4.0)
       buff-extensions (~> 0.3)
       varia_model (~> 0.1)
     buff-extensions (0.5.0)
-    buff-ignore (1.1.0)
+    buff-ignore (1.1.1)
     buff-ruby_engine (0.1.0)
     buff-shell_out (0.1.1)
       buff-ruby_engine (~> 0.1.0)
     builder (3.2.2)
-    celluloid (0.15.2)
-      timers (~> 1.1.0)
-    celluloid-io (0.15.0)
-      celluloid (>= 0.15.0)
-      nio4r (>= 0.5.0)
+    celluloid (0.14.1)
+      timers (>= 1.0.0)
+    celluloid-io (0.14.1)
+      celluloid (>= 0.14.1)
+      nio4r (>= 0.4.5)
     chef (11.6.2)
       erubis (~> 2.7)
       highline (~> 1.6, >= 1.6.9)
@@ -45,6 +49,10 @@ GEM
       ohai (>= 0.6.0, < 7.0.0)
       rest-client (>= 1.0.4, < 1.7.0)
       yajl-ruby (~> 1.1)
+    chozo (0.6.1)
+      activesupport (>= 3.2.0)
+      hashie (>= 2.0.2)
+      multi_json (>= 1.3.0)
     digital_ocean (1.2.0)
       faraday (~> 0.8.7)
       faraday_middleware (~> 0.9.0)
@@ -84,21 +92,22 @@ GEM
     httpclient (2.3.4.1)
     httpi (0.9.7)
       rack
+    i18n (0.6.9)
     ipaddress (0.8.0)
     json (1.7.7)
-    knife-digital_ocean (0.2.0)
+    knife-digital_ocean (0.3.0)
       chef (>= 10.18)
       digital_ocean (~> 1.2.0)
       highline
     knife-ec2 (0.6.6)
       chef (>= 0.10.10)
       fog (~> 1.15.0)
       knife-windows (>= 0.5.12)
-    knife-rackspace (0.8.1)
+    knife-rackspace (0.8.4)
       chef (>= 0.10.10)
       fog (~> 1.12)
       knife-windows
-    knife-windows (0.5.12)
+    knife-windows (0.5.14)
       em-winrm (= 0.5.4)
     little-plugger (1.1.3)
     logging (1.8.1)
@@ -112,7 +121,7 @@ GEM
     mixlib-cli (1.3.0)
     mixlib-config (1.1.2)
     mixlib-log (1.6.0)
-    mixlib-shellout (1.2.0)
+    mixlib-shellout (1.3.0)
     multi_json (1.8.2)
     multipart-post (1.2.0)
     net-http-persistent (2.9)
@@ -139,24 +148,26 @@ GEM
     rack (1.5.2)
     rash (0.4.0)
       hashie (~> 2.0.0)
+    rbzip2 (0.2.0)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     retryable (1.3.3)
-    ridley (1.7.1)
+    ridley (1.5.3)
       addressable
       buff-config (~> 0.2)
       buff-extensions (~> 0.3)
       buff-ignore (~> 1.1)
       buff-shell_out (~> 0.1)
-      celluloid (~> 0.15)
-      celluloid-io (~> 0.15)
+      celluloid (~> 0.14.0)
+      celluloid-io (~> 0.14.0)
       erubis
       faraday (>= 0.8.4)
       hashie (>= 2.0.2)
       json (>= 1.7.7)
       mixlib-authentication (>= 1.3.0)
       net-http-persistent (>= 2.8)
       net-ssh
+      nio4r (>= 0.5.0)
       retryable
       solve (>= 0.4.4)
       varia_model (~> 0.1)
@@ -171,7 +182,7 @@ GEM
       nokogiri (>= 1.4.0)
       nori (~> 1.0)
       wasabi (~> 1.0)
-    solve (0.8.1)
+    solve (0.8.2)
     systemu (2.5.2)
     thor (0.18.1)
     timers (1.1.0)
@@ -195,7 +206,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  berkshelf (= 3.0.0.beta3)
+  berkshelf (~> 2.0.10)
   chef
   foodcritic
   knife-digital_ocean

Makefile

@@ -3,17 +3,17 @@ all:
 
 db:
 	python manage.py syncdb --noinput
-	python manage.py migrate
+	python manage.py migrate --noinput
 
 test:
-	python manage.py test api cm provider web
+	python -Wall manage.py test --noinput api cm provider web
 
 coverage:
-	coverage run manage.py test api cm provider web
+	coverage run manage.py test --noinput api cm provider web
 	coverage html
 
 test_client:
-	python -m unittest discover client.tests
+	python -Wall -m unittest discover client.tests
 
 flake8:
 	flake8

api/migrations/0001_initial.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-import datetime
+from south.utils import datetime_utils as datetime
 from south.db import db
 from south.v2 import SchemaMigration
 from django.db import models
@@ -415,15 +415,15 @@ def backwards(self, orm):
             'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
             'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
             'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
-            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Group']"}),
             u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
             'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
             'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
             'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
             'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
             'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
-            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': u"orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "u'user_set'", 'blank': 'True', 'to': u"orm['auth.Permission']"}),
             'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
         },
         u'contenttypes.contenttype': {

api/models.py

@@ -83,7 +83,6 @@ def delete(self, *args, **kwargs):
         self.owner.publish()
 
 
-@python_2_unicode_compatible
 class ProviderManager(models.Manager):
     """Manage database interactions for :class:`Provider`."""
 
@@ -126,7 +125,6 @@ def __str__(self):
         return "{}-{}".format(self.id, self.get_type_display())
 
 
-@python_2_unicode_compatible
 class FlavorManager(models.Manager):
     """Manage database interactions for :class:`Flavor`."""
 
@@ -187,19 +185,33 @@ def flat(self):
                 'nodes': self.nodes}
 
     def build(self):
-        tasks.build_formation.delay(self).wait()
+        return
 
     def destroy(self, *args, **kwargs):
-        tasks.destroy_formation.delay(self).wait()
+        app_tasks = [tasks.destroy_app.si(a) for a in self.app_set.all()]
+        node_tasks = [tasks.destroy_node.si(n) for n in self.node_set.all()]
+        layer_tasks = [tasks.destroy_layer.si(l) for l in self.layer_set.all()]
+        group(app_tasks + node_tasks).apply_async().join()
+        group(layer_tasks).apply_async().join()
+        CM.purge_formation(self.flat())
+        self.delete()
+        tasks.converge_controller.apply_async().wait()
 
     def publish(self):
         data = self.calculate()
         CM.publish_formation(self.flat(), data)
         return data
 
-    def converge(self, **kwargs):
+    def converge(self, controller=False, **kwargs):
         databag = self.publish()
-        tasks.converge_formation.delay(self).wait()
+        nodes = self.node_set.all()
+        subtasks = []
+        for n in nodes:
+            subtask = tasks.converge_node.si(n)
+            subtasks.append(subtask)
+        if controller is True:
+            subtasks.append(tasks.converge_controller.si())
+        group(*subtasks).apply_async().join()
         return databag
 
     def calculate(self):
@@ -292,7 +304,6 @@ def destroy(self):
         return tasks.destroy_layer.delay(self).wait()
 
 
-@python_2_unicode_compatible
 class NodeManager(models.Manager):
 
     def new(self, formation, layer, fqdn=None):
@@ -462,10 +473,11 @@ def build(self):
         Release.objects.create(
             version=1, owner=self.owner, app=self, config=config, build=build)
         self.formation.publish()
-        tasks.build_app.delay(self).wait()
 
     def destroy(self):
-        tasks.destroy_app.delay(self).wait()
+        CM.purge_app(self.flat())
+        self.delete()
+        self.formation.publish()
 
     def publish(self):
         """Publish the application to configuration management"""
@@ -519,6 +531,7 @@ def logs(self):
 
     def run(self, command):
         """Run a one-off command in an ephemeral app container."""
+        # TODO: add support for interactive shell
         nodes = self.formation.node_set.filter(layer__runtime=True).order_by('?')
         if not nodes:
             raise EnvironmentError('No nodes available to run command')
@@ -527,17 +540,13 @@ def run(self, command):
         # prepare ssh command
         version = release.version
         docker_args = ' '.join(
-            ['-v',
-             '/opt/deis/runtime/slugs/{app_id}-{version}/app:/app'.format(**locals()),
-             release.build.image])
-        base_cmd = "export HOME=/app; cd /app && for profile in " \
-                   "`find /app/.profile.d/*.sh -type f`; do . $profile; done"
-        command = "/bin/sh -c '{base_cmd} && {command}'".format(**locals())
+            ['-a', 'stdout', '-a', 'stderr',
+             '-v', '/opt/deis/runtime/slugs/{app_id}-v{version}:/app'.format(**locals()),
+             'deis/slugrunner'])
         command = "sudo docker run {docker_args} {command}".format(**locals())
         return node.run(command)
 
 
-@python_2_unicode_compatible
 class ContainerManager(models.Manager):
 
     def scale(self, app, structure, **kwargs):
@@ -733,7 +742,7 @@ def push(cls, push):
         # create the build
         new_build = cls.objects.create(**push)
         # send a release signal
-        release_signal.send(sender=push, build=new_build, app=app, user=user)
+        release_signal.send(sender=user, build=new_build, app=app, user=user)
         # see if we need to scale an initial web container
         if len(app.formation.node_set.filter(layer__runtime=True)) > 0 and \
            len(app.container_set.filter(type='web')) < 1:

api/serializers.py

@@ -160,7 +160,7 @@ class AppSerializer(serializers.ModelSerializer):
 
     owner = serializers.Field(source='owner.username')
     id = serializers.SlugField(default=utils.generate_app_name)
-    formation = OwnerSlugRelatedField(slug_field='id')
+    formation = OwnerSlugRelatedField(slug_field='id', required=False)
 
     class Meta:
         """Metadata options for a :class:`AppSerializer`."""