Full Vagrant provider. Squashed commits

Author: tombh

.gitignore

@@ -50,5 +50,18 @@ htmlcov/
 .ruby-version
 venv/
 .vagrant
+Vagrantfile.local
 *.swp
+.bundle
 
+# Chef setup
+.chef
+contrib/vagrant/knife-config/admin.pem
+contrib/vagrant/knife-config/chef-validator.pem
+contrib/vagrant/nodes/
+
+# Broken symlinks that appear when mounting the codebase onto a vagrant VM
+# See Vagrantfile.local.example
+logs
+static
+venv

Gemfile

@@ -2,7 +2,7 @@ source 'https://rubygems.org'
 
 gem 'chef'
 gem 'foodcritic'
-gem 'berkshelf'
+gem 'berkshelf', '3.0.0.beta3' # needed for --ssl-verify=false support
 
 gem 'knife-ec2'
 gem 'knife-rackspace'

Gemfile.lock

@@ -49,10 +49,6 @@ GEM
       ohai (>= 0.6.0, < 7.0.0)
       rest-client (>= 1.0.4, < 1.7.0)
       yajl-ruby (~> 1.1)
-    chozo (0.6.1)
-      activesupport (>= 3.2.0)
-      hashie (>= 2.0.2)
-      multi_json (>= 1.3.0)
     digital_ocean (1.2.0)
       faraday (~> 0.8.7)
       faraday_middleware (~> 0.9.0)
@@ -92,7 +88,6 @@ GEM
     httpclient (2.3.4.1)
     httpi (0.9.7)
       rack
-    i18n (0.6.5)
     ipaddress (0.8.0)
     json (1.7.7)
     knife-digital_ocean (0.2.0)
@@ -206,7 +201,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
-  berkshelf
+  berkshelf (= 3.0.0.beta3)
   chef
   foodcritic
   knife-digital_ocean

Vagrantfile

@@ -1,35 +1,47 @@
-# -*- mode: ruby -*-
-# vi: set ft=ruby :
 
-VAGRANTFILE_API_VERSION = "2"
-
-Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
-
-  # Give each controller and node additional memory
-  config.vm.provider :virtualbox do |v|
-    v.customize ["modifyvm", :id, "--memory", 2048]
-  end
+Vagrant.configure("2") do |config|
+  config.vm.box = "deis-node"
 
-  # Deis Nodes
+  # This is the vanilla Ubunutu 12.04 Precise box. It's about 350MB
   config.vm.box_url = "https://s3-us-west-2.amazonaws.com/opdemand/deis-node.box"
-  config.vm.box = "deis-node"
 
-  # Deis Controller
-  config.vm.define "deis-controller", primary: true do |controller|
-    controller.vm.hostname = "deis-controller"
-    controller.vm.network "private_network", ip: "192.168.61.100"
-  end
+  # This is a premade Deis controller box, probably 12.04, need to check. It's about 1.1GB
+  # config.vm.box_url = "https://s3-us-west-2.amazonaws.com/opdemand/deis-controller.box"
 
-  # Node 1
-  config.vm.define "deis-node-1" do |node1|
-    node1.vm.hostname = "deis-node-1"
-    node1.vm.network "private_network", ip: "192.168.61.101"
-  end
+  # Avahi-daemon will broadcast the server's address as deis-controller.local
+  config.vm.host_name = "deis-controller"
 
-  # Node 2
-  config.vm.define "deis-node-2" do |node2|
-    node2.vm.hostname = "deis-node-2"
-    node2.vm.network "private_network", ip: "192.168.61.102"
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network. IP will be fetched via DCHP and associated to 'chefserver.local'
+  # using avahi-daemon
+  config.vm.network :public_network
+
+  # Chef Server requires at least 1G of RAM to install.
+  # You may be able to run it with less once it's installed.
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", "1024"]
   end
 
+  # 'deis provider:discover' detects the host machine's user and IP address, however, that command cannot
+  # be guareteed to run inside the deis codebase. Therefore we can't use that opportunity to discover
+  # the path of the codebase on the host machine. Therefore we do it now as this Vagrantfile has to exist
+  # inside the codebase.
+  nodes_dir = File.dirname(__FILE__) + '/contrib/vagrant/nodes'
+
+  config.vm.provision :shell, inline: <<-SCRIPT
+    # Avahi-daemon broadcasts the machine's hostname to local DNS.
+    # Therefore 'deis-controller.local' in this case.
+    sudo service avahi-daemon restart
+    # Make a record of where the deis code base is on the host machine
+    echo "#{nodes_dir}" > /home/vagrant/.host_nodes_dir
+  SCRIPT
 end
+
+# If you want to do some funky custom stuff to your box, but don't want those things tracked by git,
+# add a Vagrantfile.local and it will be included. You can use the exact same syntax as above. For
+# example you could mount your dev version of deis onto the VM and hack live on the VM;
+# `config.vm.share_folder "deis", "/opt/deis", "~/myworkspace/deis"
+# Or if you're low on RAM you can boot the VM with less RAM. Note that at least 1GB is needed for
+# installation, but you may be able to get away with 512MB once everything is installed.
+load "Vagrantfile.local" if File.exists? "Vagrantfile.local"

Vagrantfile.local.example

@@ -0,0 +1,53 @@
+=begin
+This file augments the standard Vagrantfile with some developer-specific settings.
+You can override any settings you want. This file is in .gitignore, so can be specific
+to your own environment.
+
+By default this file mounts your local development version of the Deis codebase onto the
+Deis Controller. It does this by mounting your code onto the VM at /vagrant, then moving
+the original installed code base to /opt/deis/controller_ORIG and symlinking
+/opt/deis/controller to the mounted code at /vagrant.
+
+Because the deployed environment creates some extra files, like /deis/local_settings.py
+for example, these need to be then symlinked into the symlinked mount. Sound complicated?
+Yo dawg, I heard you like symlinks.
+=end
+
+HOST_CODE_PATH = '/opt/deis/controller'
+HOST_BAK_PATH = "#{HOST_CODE_PATH}_ORIG"
+
+# These paths are either in .gitignore or created after installation.
+# Therefore they need to be linked into the dev codebase.
+symlinks = [
+  'deis/local_settings.py',
+  '.secret_key',
+  'logs',
+  'static',
+  'venv'
+].map { |path|
+  "ln -s #{HOST_BAK_PATH}/#{path} #{HOST_CODE_PATH}/#{path}"
+}.join("\n")
+
+Vagrant.configure("2") do |config|
+
+  config.vm.provider :virtualbox do |vb|
+    #vb.customize ["modifyvm", :id, "--memory", "512"]
+  end
+
+  # Mount the entire Deis codebase onto the VM at /vagrant
+  # The first param is relative to the location of this file
+  config.vm.synced_folder "../../../", "/vagrant", owner: 'deis', group: 'deis'
+
+  config.vm.provision :shell, inline: <<-SCRIPT
+    if [ ! -d #{HOST_BAK_PATH} ]; then
+      mv #{HOST_CODE_PATH} #{HOST_BAK_PATH}
+      ln -s /vagrant #{HOST_CODE_PATH}
+
+      # Paths added after installation need linking into the codebase
+      #{symlinks}
+
+      chown -R deis:deis #{HOST_CODE_PATH}
+    fi
+  SCRIPT
+
+end

api/models.py

@@ -110,6 +110,7 @@ class Provider(UuidAuditedModel):
         ('rackspace', 'Rackspace Open Cloud'),
         ('static', 'Static Node'),
         ('digitalocean', 'Digital Ocean'),
+        ('vagrant', 'Local Vagrant VMs'),
     )
 
     owner = models.ForeignKey(settings.AUTH_USER_MODEL)

client/deis.py

@@ -1754,6 +1754,51 @@ def providers_discover(self, args):
         else:
             print('No Digitalocean credentials discovered.')
 
+        # Check for locally booted Deis Controller VM
+        try:
+            running_vms = subprocess.check_output(
+                    ['vboxmanage', 'list', 'runningvms'],
+                    stderr=subprocess.PIPE)
+        except subprocess.CalledProcessError:
+            running_vms = ""
+        # Vagrant internally names a running VM using the folder name in which the Vagrantfile resides, eg;
+        # my-deis-code-folder_default_1383326629
+        deis_codebase_folder = self._session.git_root().split('/')[-1]
+        if deis_codebase_folder in running_vms:
+            print("Detected locally running Deis Controller VM")
+            # In order for the Controller to be able to boot Vagrant VMs it needs to run commands on the host machine.
+            # It does this via an SSH server. In order to access that server we need to send the current user's name
+            # and IP address.
+            try:
+                user = subprocess.check_output(
+                    "whoami",
+                    stderr=subprocess.PIPE
+                ).strip()
+                ip = subprocess.check_output(
+                    "/sbin/ifconfig | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | head -n1",
+                    stderr=subprocess.PIPE,
+                    shell=True
+                ).strip()
+            except subprocess.CalledProcessError:
+                print("Error detecting username and host IP address.")
+                sys.exit(1)
+            creds = {
+                'user': user,
+                'ip': ip
+            }
+            body = {'creds': json.dumps(creds)}
+            sys.stdout.write('Activating Vagrant as a provider... ')
+            sys.stdout.flush()
+            response = self._dispatch('patch', '/api/providers/vagrant',
+                                      json.dumps(body))
+            if response.status_code == requests.codes.ok:  # @UndefinedVariable
+                print('done')
+            else:
+                raise ResponseError(response)
+        else:
+            print("No Vagrant VMs detected")
+
+
     def providers_info(self, args):
         """
         Print information about a specific provider

contrib/vagrant/README.md

@@ -1,35 +1,116 @@
-Provision a Deis Controller on Vagrant
-======================================
+Local development workflow for Deis
+================================================================
 
-This document describes how to set up a Deis controller and two nodes with
-Vagrant for testing.
+This document suggests some ways that might make developing Deis easier and cheaper. You don't have to follow
+them all.
 
-1. Install VirtualBox version 4.2.18. (Vagrant does not support version 4.3.)
-Then start VirtualBox and install the VirtualBox Extension Pack for 4.2.18.
+1. You'll need VirtualBox >= `4.2.18`. We recommend installing Vagrant with their binary installer from http://downloads.vagrantup.com
+Vagrant 1.3.5 has support for VirtualBox 4.3
 
-2. Install Vagrant version 1.3.5 or later. Rather than fight Ruby dependencies,
-use a binary installer from vagrantup.com.
+2. Firstly you need to decide whether to use your own Chef Server or the free tier of the hosted enterprise 
+service from Opscode. The free tier has a limit of 5 nodes which is more then enough for development. Also
+bear in mind that a local Chef Server VM will take up at least 1GB of RAM.
 
-3. Run the provisioning script:
-```console
-$ ./contrib/vagrant/provision-vagrant-controller.sh
-```
+    **Local Chef Server**
+    * `cd [DEIS_DIR] && ln -s contrib/vagrant/knife-config .chef`
+    * `vagrant up` the chef server Vagrantfile.
+    * copy the admin.pem and validation.pem files for your own knife client
+    `scp -r root@chefserver.local:/etc/chef-server/admin.pem [DEIS_DIR]/contrib/vagrant/knife-config/`
+    `scp -r root@chefserver.local:/etc/chef-server/chef-validator.pem [DEIS_DIR]/contrib/vagrant/knife-config/`
 
-This script will:
-- Create the data bags in your Chef account to support Deis
-- Run `vagrant up` to create a Deis controller and 2 static nodes
-- Register the controller with Chef and install Deis and supporting software
+    **Hosted Chef Server**
+    * Goto https://getchef.opscode.com/signup and fill in your details.
+    * Goto https://preview.opscode.com/login and sign in to your Chef Server.
+    * Click on the 'Administration' tab and choose your organisation. There should be a tab in the sidebar that says
+    'Starter Kit'. Click it and it will start a small download.
+    * Inside the Starter Kit there is a '.chef' folder. Copy it to the root of your Deis codebase.
 
-4. Register a user
+3. Now you can follow the standard deis setup:
+  ```bash
+  bundle install # Installs gem files like the knife tool
+  berks install # Downloads the relevant cookbooks
+  # '--ssl-verify' is only needed when using a local Chef Server
+  berks upload [--ssl-verify=false] # Upload the cookbooks to the Chef Server
+  ```
 
-5. Run the script to create a static formation:
-```console
-$ ./contrib/vagrant/create-static-formation.sh
-```
+4. Use the provision script to boot the deis controller.
+    * If you don't already have the deis-node Vagrant box installed (~1GB). This step might take a long time! If for some reason 
+    you want to manually add it, use:
+    `vagrant box add deis-node https://s3-us-west-2.amazonaws.com/opdemand/deis-node.box`
+    * `cd contrib/vagrant && ./provision-controller.sh`
+    * You will be asked to add the Controller's SSH key to your local SSH server. This will allow the Controller
+    to run vagrant commands on your machine to bootstrap new nodes.
+    * If you are using a local Chef Server you will need to tell it that your new controller has permission to create
+    nodes. Use:
+    `knife client edit deis-controller`
+    and your default text editor will launch, you need to set 'admin' to 'true'.
+
+5. The Controller needs to be able to run Vagrant commands on your host machine. It does this via SSH. Therefore
+you will need a running SSH server open on port 22.
+    * On Debian-flavoured Linux you just need to;
+    `sudo apt-get install openssh-server`
+    * On Mac OSX you just need to go to **System Preferences -> Sharing** and enable 'Remote Login'.
+    * **NB** If your machine's IP changes the Controller won't be able to run commands any more. Currently you aren't informed
+    of this, you just get a 500 error from the client. If you tail `/var/log/deis/celeryd.log` though you'll know. When this
+    does happen just reupload your IP with `deis providers:discover`.
+
+6. If you want to hack on the actual codebase, you can mount your local codebase onto the VM
+   by using the custom Vagrantfile.local.
+   * `cp Vagrantfile.local.example Vagrantfile.local` (don't worry it's in .gitignore)
+   * Update the VM with `vagrant reload --provision`
+   * When mounted you can use your favourite editor to change the code _on your local machine's path_ and then run
+   `service deis-server restart` and/or `service deis-worker restart` on the VM for your changes to instantly take effect.
+   * It's worth having a read of `Vagrantfile.local.example`
+
+7. If you want to hack on the command line client (`/client/deis.py`), install your local dev version rather than
+the one from Pip.
+    * `cd deis && make install` This will symlink the dev version to your executables path.
+    * Your deis controller is available at http://deis-controller.local so you can register with;
+    `deis register http://deis-controller.local`
+
+8. Right, time to boot up some nodes!
+  * Create a foramtion with a vagrant flavour, 512MB, 1024MB and 2048MB are available.
+  `deis formations:create dev --flavor=vagrant-512 --domain=deisapp.local`
+  * Scale a node with `deis nodes:scale dev runtime=1` Be patient, this is the command that runs vagrant commands. Scaling a single node
+  can take about 5 mins.
+  * Then create and push your app as per the usual documentation.
+
+## Useful development commands
+* To use Django's manage.py:
+  * SSH in to the VM with `vagrant ssh`
+  * Switch user to deis with `sudo su deis`
+  * `cd /opt/deis/controller` and activate Venv with `. venv/bin/activate`
+  * Get a list of commands with; `./manage.py help`.
+
+* To reset the DB:
+  * On the VM run `sudo su postgres -c 'dropdb deis && createdb --encoding=utf8 --template=template0 deis'`
+  * When you restart the server with `sudo service deis-server restart` Django will reinstall the DB.
+  * You'll need to reupdate the Django's Site Object
+  `sudo su deis -c "psql deis -c \"UPDATE django_site SET domain = 'deis-controller.local', name = 'deis-controller.local' WHERE id = 1 \""`
+
+* This is useful for uploading your own local version of the cookbooks, rather than the Github versions:
+  * `knife cookbook upload deis --cookbook-path [deis-cookbook path] --force`
+  * You need to change directory structure though as knife gets the cookbook name from the folder name. So, for example I use;
+
+  ```bash
+  deis
+    |__ code
+      |__ api
+      |__ bin
+      |__ client
+      |__ cm
+      |__ ... and so on
+    |__ cookbooks
+      |__ deis
+        |__ attributes
+        |__ definitions
+        |__ recipes
+        |__ ... and so on
+  ```
 
 Notes
 -----
 
 Mac OS X: if you see an error such as
-"failed to open /dev/vboxnetctl", try restarting VirtualBox:
-sudo /Library/StartupItems/VirtualBox/VirtualBox restart
+'failed to open /dev/vboxnetctl', try restarting VirtualBox:
+sudo /Library/StartupItems/VirtualBox/VirtualBox restart