Merge branch '232-vagrant-provider-full' of https://github.com/tombh/deis into tombh-232-vagrant-provider-full

Author: mboersma

.gitignore

@@ -50,5 +50,18 @@ htmlcov/
 .ruby-version
 venv/
 .vagrant
+Vagrantfile.local
 *.swp
+.bundle
 
+# Chef setup
+.chef
+contrib/vagrant/knife-config/admin.pem
+contrib/vagrant/knife-config/chef-validator.pem
+contrib/vagrant/nodes/
+
+# Broken symlinks that appear when mounting the codebase onto a vagrant VM
+# See Vagrantfile.local.example
+logs
+static
+venv

Berksfile

@@ -1,4 +1,4 @@
-site :opscode
+source "http://api.berkshelf.com"
 
 cookbook 'apt'
 cookbook 'rsyslog'

Berksfile.lock

@@ -1,5 +1,5 @@
 {
-  "sources": {
+  "dependencies": {
     "apt": {
       "locked_version": "2.3.0"
     },
@@ -10,9 +10,9 @@
       "locked_version": "2.2.2"
     },
     "deis": {
-      "locked_version": "0.2.1",
+      "locked_version": "0.1.2",
       "git": "https://github.com/opdemand/deis-cookbook.git",
-      "ref": "617d5a2627fdee92ade9499b4a87c6ff85a99c66"
+      "ref": "3277d069b61d261b6ebcab319041442fe38a2ae3"
     }
   }
 }

Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 gem 'chef'
 gem 'foodcritic'
-gem 'berkshelf'
+gem 'berkshelf', '3.0.0.beta3' # needed for --ssl-verify=false support
 
 gem 'knife-ec2'
 gem 'knife-rackspace'

Gemfile.lock

@@ -1,25 +1,21 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (3.2.15)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
     addressable (2.3.5)
     akami (1.2.0)
       gyoku (>= 0.4.0)
       nokogiri (>= 1.4.0)
-    berkshelf (2.0.10)
-      activesupport (~> 3.2.0)
+    berkshelf (3.0.0.beta3)
       addressable (~> 2.3.4)
+      buff-config (~> 0.2)
+      buff-extensions (~> 0.4)
       buff-shell_out (~> 0.1)
-      chozo (>= 0.6.1)
       faraday (>= 0.8.5)
       hashie (>= 2.0.2)
       minitar (~> 0.5.4)
-      rbzip2 (~> 0.2.0)
       retryable (~> 1.3.3)
-      ridley (~> 1.5.0)
-      solve (>= 0.5.0)
+      ridley (~> 1.6)
+      solve (>= 0.8.0)
       thor (~> 0.18.0)
     buff-config (0.4.0)
       buff-extensions (~> 0.3)
@@ -30,11 +26,11 @@ GEM
     buff-shell_out (0.1.1)
       buff-ruby_engine (~> 0.1.0)
     builder (3.2.2)
-    celluloid (0.14.1)
-      timers (>= 1.0.0)
-    celluloid-io (0.14.1)
-      celluloid (>= 0.14.1)
-      nio4r (>= 0.4.5)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
     chef (11.6.2)
       erubis (~> 2.7)
       highline (~> 1.6, >= 1.6.9)
@@ -49,10 +45,6 @@ GEM
       ohai (>= 0.6.0, < 7.0.0)
       rest-client (>= 1.0.4, < 1.7.0)
       yajl-ruby (~> 1.1)
-    chozo (0.6.1)
-      activesupport (>= 3.2.0)
-      hashie (>= 2.0.2)
-      multi_json (>= 1.3.0)
     digital_ocean (1.2.0)
       faraday (~> 0.8.7)
       faraday_middleware (~> 0.9.0)
@@ -92,7 +84,6 @@ GEM
     httpclient (2.3.4.1)
     httpi (0.9.7)
       rack
-    i18n (0.6.5)
     ipaddress (0.8.0)
     json (1.7.7)
     knife-digital_ocean (0.2.0)
@@ -148,26 +139,24 @@ GEM
     rack (1.5.2)
     rash (0.4.0)
       hashie (~> 2.0.0)
-    rbzip2 (0.2.0)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     retryable (1.3.3)
-    ridley (1.5.3)
+    ridley (1.7.1)
       addressable
       buff-config (~> 0.2)
       buff-extensions (~> 0.3)
       buff-ignore (~> 1.1)
       buff-shell_out (~> 0.1)
-      celluloid (~> 0.14.0)
-      celluloid-io (~> 0.14.0)
+      celluloid (~> 0.15)
+      celluloid-io (~> 0.15)
       erubis
       faraday (>= 0.8.4)
       hashie (>= 2.0.2)
       json (>= 1.7.7)
       mixlib-authentication (>= 1.3.0)
       net-http-persistent (>= 2.8)
       net-ssh
-      nio4r (>= 0.5.0)
       retryable
       solve (>= 0.4.4)
       varia_model (~> 0.1)
@@ -206,7 +195,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  berkshelf
+  berkshelf (= 3.0.0.beta3)
   chef
   foodcritic
   knife-digital_ocean

Vagrantfile

@@ -1,35 +1,44 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
 
-VAGRANTFILE_API_VERSION = "2"
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-
-  # Give each controller and node additional memory
-  config.vm.provider :virtualbox do |v|
-    v.customize ["modifyvm", :id, "--memory", 2048]
-  end
+Vagrant.configure("2") do |config|
+  config.vm.box = "deis-node"
 
-  # Deis Nodes
+  # This is the vanilla Ubunutu 12.04 Precise box. It's about 350MB
   config.vm.box_url = "https://s3-us-west-2.amazonaws.com/opdemand/deis-node.box"
-  config.vm.box = "deis-node"
 
-  # Deis Controller
-  config.vm.define "deis-controller", primary: true do |controller|
-    controller.vm.hostname = "deis-controller"
-    controller.vm.network "private_network", ip: "192.168.61.100"
-  end
+  # This is a premade Deis controller box, probably 12.04, need to check. It's about 1.1GB
+  # config.vm.box_url = "https://s3-us-west-2.amazonaws.com/opdemand/deis-controller.box"
 
-  # Node 1
-  config.vm.define "deis-node-1" do |node1|
-    node1.vm.hostname = "deis-node-1"
-    node1.vm.network "private_network", ip: "192.168.61.101"
-  end
+  # Avahi-daemon will broadcast the server's address as deis-controller.local
+  config.vm.host_name = "deis-controller"
 
-  # Node 2
-  config.vm.define "deis-node-2" do |node2|
-    node2.vm.hostname = "deis-node-2"
-    node2.vm.network "private_network", ip: "192.168.61.102"
+  # IP will be associated to 'deis-controller.local' using avahi-daemon
+  config.vm.network :private_network, ip: "192.168.61.100"
+
+  # The Deis Controller requires at least 1G of RAM to install.
+  # You may be able to run it with less once it's installed.
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", "1024"]
   end
 
+  # 'deis provider:discover' detects the host machine's user and IP address, however, that command cannot
+  # be guareteed to run inside the deis codebase. Therefore we can't use that opportunity to discover
+  # the path of the codebase on the host machine. Therefore we do it now as this Vagrantfile has to exist
+  # inside the codebase.
+  nodes_dir = File.dirname(__FILE__) + '/contrib/vagrant/nodes'
+
+  config.vm.provision :shell, inline: <<-SCRIPT
+    # Avahi-daemon broadcasts the machine's hostname to local DNS.
+    # Therefore 'deis-controller.local' in this case.
+    sudo service avahi-daemon restart
+    # Make a record of where the deis code base is on the host machine
+    echo "#{nodes_dir}" > /home/vagrant/.host_nodes_dir
+  SCRIPT
 end
+
+# If you want to do some funky custom stuff to your box, but don't want those things tracked by git,
+# add a Vagrantfile.local and it will be included. You can use the exact same syntax as above. For
+# example you could mount your dev version of deis onto the VM and hack live on the VM;
+# `config.vm.share_folder "deis", "/opt/deis", "~/myworkspace/deis"
+# Or if you're low on RAM you can boot the VM with less RAM. Note that at least 1GB is needed for
+# installation, but you may be able to get away with 512MB once everything is installed.
+load "Vagrantfile.local" if File.exists? "Vagrantfile.local"

Vagrantfile.local.example

@@ -0,0 +1,53 @@
+=begin
+This file augments the standard Vagrantfile with some developer-specific settings.
+You can override any settings you want. This file is in .gitignore, so can be specific
+to your own environment.
+
+By default this file mounts your local development version of the Deis codebase onto the
+Deis Controller. It does this by mounting your code onto the VM at /vagrant, then moving
+the original installed code base to /opt/deis/controller_ORIG and symlinking
+/opt/deis/controller to the mounted code at /vagrant.
+
+Because the deployed environment creates some extra files, like /deis/local_settings.py
+for example, these need to be then symlinked into the symlinked mount. Sound complicated?
+Yo dawg, I heard you like symlinks.
+=end
+
+HOST_CODE_PATH = '/opt/deis/controller'
+HOST_BAK_PATH = "#{HOST_CODE_PATH}_ORIG"
+
+# These paths are either in .gitignore or created after installation.
+# Therefore they need to be linked into the dev codebase.
+symlinks = [
+  'deis/local_settings.py',
+  '.secret_key',
+  'logs',
+  'static',
+  'venv'
+].map { |path|
+  "ln -s #{HOST_BAK_PATH}/#{path} #{HOST_CODE_PATH}/#{path}"
+}.join("\n")
+
+Vagrant.configure("2") do |config|
+
+  config.vm.provider :virtualbox do |vb|
+    #vb.customize ["modifyvm", :id, "--memory", "512"]
+  end
+
+  # Mount the entire Deis codebase onto the VM at /vagrant
+  # The first param is relative to the location of this file
+  config.vm.synced_folder ".", "/vagrant", owner: 'deis', group: 'deis'
+
+  config.vm.provision :shell, inline: <<-SCRIPT
+    if [ ! -d #{HOST_BAK_PATH} ]; then
+      mv #{HOST_CODE_PATH} #{HOST_BAK_PATH}
+      ln -s /vagrant #{HOST_CODE_PATH}
+
+      # Paths added after installation need linking into the codebase
+      #{symlinks}
+
+      chown -R deis:deis #{HOST_CODE_PATH}
+    fi
+  SCRIPT
+
+end

api/models.py

@@ -110,6 +110,7 @@ class Provider(UuidAuditedModel):
         ('rackspace', 'Rackspace Open Cloud'),
         ('static', 'Static Node'),
         ('digitalocean', 'Digital Ocean'),
+        ('vagrant', 'Local Vagrant VMs'),
     )
 
     owner = models.ForeignKey(settings.AUTH_USER_MODEL)

client/deis.py

@@ -1661,7 +1661,7 @@ def providers(self, args):
         """
         return self.providers_list(args)
 
-    def providers_create(self, args):
+    def providers_create(self, args):  # noqa
         """
         Create a provider for use by Deis
 
@@ -1691,7 +1691,7 @@ def providers_create(self, args):
         else:
             raise ResponseError(response)
 
-    def providers_discover(self, args):
+    def providers_discover(self, args):  # noqa
         """
         Discover and update provider credentials
 
@@ -1729,6 +1729,51 @@ def providers_discover(self, args):
             else:
                 print("No {} credentials discovered.".format(provider))
 
+        # Check for locally booted Deis Controller VM
+        try:
+            running_vms = subprocess.check_output(
+                ['vboxmanage', 'list', 'runningvms'],
+                stderr=subprocess.PIPE
+            )
+        except subprocess.CalledProcessError:
+            running_vms = ""
+        # Vagrant internally names a running VM using the folder name in which the Vagrantfile
+        # resides, eg; my-deis-code-folder_default_1383326629
+        deis_codebase_folder = self._session.git_root().split('/')[-1]
+        if deis_codebase_folder in running_vms:
+            print("Detected locally running Deis Controller VM")
+            # In order for the Controller to be able to boot Vagrant VMs it needs to run commands
+            # on the host machine. It does this via an SSH server. In order to access that server
+            # we need to send the current user's name and host.
+            try:
+                user = subprocess.check_output(
+                    "whoami",
+                    stderr=subprocess.PIPE
+                ).strip()
+                hostname = subprocess.check_output(
+                    "hostname",
+                    stderr=subprocess.PIPE,
+                    shell=True
+                ).strip()
+            except subprocess.CalledProcessError:
+                print("Error detecting username and host address.")
+                sys.exit(1)
+            creds = {
+                'user': user,
+                'host': hostname + ".local"
+            }
+            body = {'creds': json.dumps(creds)}
+            sys.stdout.write('Activating Vagrant as a provider... ')
+            sys.stdout.flush()
+            response = self._dispatch('patch', '/api/providers/vagrant',
+                                      json.dumps(body))
+            if response.status_code == requests.codes.ok:  # @UndefinedVariable
+                print('done')
+            else:
+                raise ResponseError(response)
+        else:
+            print("No Vagrant VMs detected")
+
     def providers_info(self, args):
         """
         Print information about a specific provider