feat(contrib/vagrant): support multiple-VM environment

This commit modifies the Vagrantfile and Makefile in contrib/vagrant
to support an arbitrary number of Deis cluster machines.

Author: carmstrong

contrib/vagrant/Makefile

@@ -0,0 +1,51 @@
+#
+# Deis Makefile
+#
+
+ifndef $(DEIS_NUM_INSTANCES)
+    DEIS_NUM_INSTANCES = 3
+endif
+
+define ssh_all
+  i=1 ; while [[ $$i -le $(DEIS_NUM_INSTANCES) ]] ; do \
+      vagrant ssh deis-$$i -c $(1) ; \
+      ((i = i + 1)) ; \
+  done
+endef
+
+# ordered list of deis components
+COMPONENTS=registry logger database cache controller builder router
+
+all: build run
+
+test_client:
+	python -m unittest discover client.tests
+
+pull:
+	$(call ssh_all,'for c in $(COMPONENTS); do docker pull deis/$$c; done')
+
+build:
+	$(call ssh_all,'cd share && for c in $(COMPONENTS); do cd $$c && docker build -t deis/$$c . && cd ..; done')
+
+install:
+	for c in $(COMPONENTS); do fleetctl enable ../../$$c/systemd/*; done
+
+uninstall: stop
+	for c in $(COMPONENTS); do fleetctl disable ../../$$c/systemd/*; done
+
+start:
+	for c in $(COMPONENTS); do fleetctl start ../../$$c/systemd/*; done
+
+stop:
+	for c in $(COMPONENTS); do fleetctl stop ../../$$c/systemd/*; done
+
+restart:
+	for c in $(COMPONENTS); do fleetctl restart ../../$$c/systemd/*; done
+
+run: install restart
+
+clean: uninstall
+	$(call ssh_all,'cd share && for c in $(COMPONENTS); do docker rm -f deis-$$c; done')
+
+full-clean: clean
+	$(call ssh_all,'cd share && for c in $(COMPONENTS); do docker rmi deis-$$c; done')

contrib/vagrant/README.md

@@ -1,126 +1,39 @@
-Local development workflow for Deis
-================================================================
-
-This document suggests some ways that might make developing Deis easier and cheaper. You don't have to follow
-them all.
-
-1. You'll need VirtualBox >= `4.2.18`. We recommend installing Vagrant with their binary installer from http://downloads.vagrantup.com
-Vagrant 1.3.5 has support for VirtualBox 4.3
-
-2. Firstly you need to decide whether to use your own Chef Server or the free tier of the hosted enterprise
-service from Opscode. The free tier has a limit of 5 nodes which is more then enough for development. Also
-bear in mind that a local Chef Server VM will take up at least 1GB of RAM.
-
-    **Local Chef Server**
-    * `cd [DEIS_DIR] && ln -s contrib/vagrant/knife-config .chef`
-    * `vagrant up` the chef server Vagrantfile.
-    * copy the admin.pem and validation.pem files for your own knife client
-    `scp -r root@chefserver.local:/etc/chef-server/admin.pem [DEIS_DIR]/contrib/vagrant/knife-config/`
-    `scp -r root@chefserver.local:/etc/chef-server/chef-validator.pem [DEIS_DIR]/contrib/vagrant/knife-config/`
-
-    **Hosted Chef Server**
-    * Goto https://getchef.opscode.com/signup and fill in your details.
-    * Goto https://manage.opscode.com/login and sign in to your Chef Server.
-    * Click on the 'Administration' tab and choose your organisation. There should be a tab in the sidebar that says
-    'Starter Kit'. Click it and it will start a small download.
-    * Inside the Starter Kit there is a '.chef' folder. Copy it to the root of your Deis codebase.
-
-3. Now you can follow the standard deis setup:
-  * If you're running a local chef server, you should adjust the `Gemfile` and make sure the version of berkshelf is 3.0.x. This is needed for the `--ssl-verify` option to work correctly.
-  ```bash
-  bundle install # Installs gem files like the knife tool
-  berks install # Downloads the relevant cookbooks
-  # '--ssl-verify' is only needed when using a local Chef Server
-  berks upload [--ssl-verify=false] # Upload the cookbooks to the Chef Server
-  ```
-
-4. The Controller needs to be able to run Vagrant commands on your host machine. It does this via SSH. Therefore
-you will need a running SSH server open on port 22 and a means to broadcast your hostname to local DNS.
-    * On Debian-flavoured Linux you just need to;
-    `sudo apt-get install openssh-server`
-    * On Mac OSX you just need to go to **System Preferences -> Sharing** and enable 'Remote Login'.
-    * [Mac OSX user's should already be broadcasting their hostname](http://support.apple.com/kb/ht3473).
-    * Linux user's will need to install avahi-daemon, so that their machine is accessible via
-    [hostname].local. Eg; `sudo apt-get install avahi-daemon`.
-
-5. Creating the Deis Controller.
-    * Run `./contrib/vagrant/provision-vagrant-controller.sh`
-    * You may need to prepend the command with `bundle exec` depending on your Ruby setup.
-    * When running for the first time you will be asked to add the Controller's SSH key to your local SSH server.
-    This will allow the Controller to run vagrant commands on your machine to bootstrap new nodes.
-    * You need to tell the Chef Server that your new Controller has permission to create
-    and delete nodes. Use:
-      * For a local Chef Server just type `knife client edit deis-controller` and your default text
-      editor will launch, you need to set 'admin' to 'true'.
-      * For Hosted Chef, log in to https://manage.opscode.com/. Go to the
-      Administration tab, click on the "Groups" entry to the left, then the "admins" entry
-      under "All Groups". Choose the "Permissions" tab and click the "+ Add" button, then
-      type in "deis-controller" and add it. Assign all permissions to the "deis-controller"
-      client object.
-
-6. If you want to hack on the command line client (`client/deis.py`), install your local dev version rather than
-the one from Pip.
-    * `cd deis && make install` This installs the client into your executables path.
-    * `sudo rm /usr/local/bin/deis && sudo ln ./deis.py /usr/local/bin/deis` This will symlink the dev version to your executables path.
-    * Your deis controller is available at http://deis-controller.local:8000 so you can register with;
-    `deis register http://deis-controller.local:8000`
-
-7. Right, time to boot up some nodes!
-  * Create a formation with a vagrant flavour, 512MB, 1024MB and 2048MB are available.
-  `deis formations:create dev --flavor=vagrant-512 --domain=deisapp.local`
-  * Scale a node with `deis nodes:scale dev runtime=1` Be patient, this is the command that runs vagrant commands. Scaling a single node
-  can take over 15 mins.
-  * Then create and push your app as per the usual documentation.
-
-## Useful development commands
-* To get a shell session to a running container use the `dsh` command on the VM. Usage:
-  * `dsh deis-builder`
-  * `dsh deis-builder /bin/ls` Note the absolute path
-  * `echo 'ls' | dsh deis-builder` Note no need for path when piping
-
-* To use Django's manage.py:
-  * SSH in to the VM with `vagrant ssh`
-  * Use `dsh deis-controller`
-  * Get into the Django server's path `cd /app/deis/controller`
-  * Get a list of commands with; `./manage.py help`
-
-* Django's native web admin interface is available at http://deis-controller.local:8000/admin/
-You can add and update all of the models from there.
-
-* To reset the DB:
-  * There is a script at contrib/vagrant/util/reset-db.sh that resets the DB and installs some basic fixtures.
-  It should be run from your host machine.
-  * It installs a formation named 'dev' and a super user with username 'devuser' and password 'devpass'.
-
-* This is useful for uploading your own local version of the cookbooks, rather than the Github versions:
-  * `knife cookbook upload deis --cookbook-path [deis-cookbook path] --force`
-  * You need to change directory structure though as knife gets the cookbook name from the folder name. So, for example I use;
-
-  ```bash
-  deis
-    |__ code
-      |__ api
-      |__ bin
-      |__ client
-      |__ cm
-      |__ ... and so on
-    |__ cookbooks
-      |__ deis
-        |__ attributes
-        |__ definitions
-        |__ recipes
-        |__ ... and so on
-  ```
-
-* Resolving submodule conflicts when merging master into your dev branch can be troublesome.
-You might have luck with `git submodule foreach git checkout master && git submodule foreach git pull --rebase`
-
-* There is a sample `Vagrantfile.local.example` that can be useful if you want any personal customisations
-to your vagrant setup, like using less RAM for post-installation boots.
-
-Notes
------
-
-Mac OS X: if you see an error such as
-'failed to open /dev/vboxnetctl', try restarting VirtualBox:
-sudo /Library/StartupItems/VirtualBox/VirtualBox restart
+# Deis cluster local test environment
+
+The Vagrantfile and Makefile provided in this directory will launch a multi-host Deis CoreOS cluster that runs locally.
+
+## Prerequisites
+On your workstation:
+* Install [Vagrant](http://www.vagrantup.com/downloads.html) and [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
+* Install [Go](http://golang.org/doc/install) and configure your GOPATH, if necessary
+* Install the fleetctl client: `go get github.com/coreos/fleetctl`
+* Set the `DEIS_NUM_INSTANCES` environment variable if you'd like more (or less) than the default of 3 machines to test:
+  * `export DEIS_NUM_INSTANCES=5`
+* Configure the fleetctl client to tunnel through one of the VMs:
+  * `export FLEETCTL_TUNNEL=172.17.8.100`
+  * (Note that IP addressing for the VMs starts at .100, but you can connect to any VM in the cluster)
+
+## Launching Deis
+Follow the normal instructions in the [README](../../README.md) for launching and using Deis, with the caveat that
+commands should be run in this directory to ensure that the proper Vagrantfile and Makefile is used. In this directory
+pull/build commands are run in each VM, and service starting/stopping are run with `fleetctl` and affect the entire cluster.
+
+## Testing ideas
+### Stop a container service
+Logging into one of the CoreOS machines and stopping a container service should cause the same component on another CoreOS
+host to take over as master
+
+### Stop a VM
+Similarly, bringing down a VM should enable the services on another VM to take over as master
+
+## Useful references
+### systemd services
+These systemd services run the various containers which compose Deis, and can be stopped on a machine with `sudo systemctl stop servicename`.
+* deis-builder.service
+* deis-cache.service
+* deis-controller.service
+* deis-database.service
+* deis-discovery.service
+* deis-logger.service
+* deis-registry.service
+* deis-router.service

contrib/vagrant/Vagrantfile

@@ -1,35 +1,53 @@
-Vagrant.configure("2") do |config|
-  config.vm.box = "deis-server"
+# -*- mode: ruby -*-
+# # vi: set ft=ruby :
 
-  # Ubuntu 12.04.3 LTS base with 3.8 kernel (ready for Docker)
-  config.vm.box_url = "https://s3-us-west-2.amazonaws.com/opdemand/ubuntu-12.04.3-amd64-vbox.box"
+require_relative '../coreos/override-plugin.rb'
 
-  # Avahi-daemon will broadcast the server's address as deis-controller.local
-  config.vm.host_name = "deis-controller"
+DEIS_NUM_INSTANCES = (ENV['DEIS_NUM_INSTANCES'].to_i > 0 && ENV['DEIS_NUM_INSTANCES'].to_i) || 3
 
-  # IP will be associated to 'deis-controller.local' using avahi-daemon
-  config.vm.network :private_network, ip: "192.168.61.100"
+Vagrant.configure("2") do |config|
+  config.vm.box = "coreos-alpha"
+  config.vm.box_url = "http://storage.core-os.net/coreos/amd64-usr/alpha/coreos_production_vagrant.box"
 
-  config.vm.provider :virtualbox do |vb|
-    # The Deis Controller requires at least 2G of RAM to install.
-    vb.customize ["modifyvm", :id, "--memory", "2048"]
-    
-    # Displays a friendly name in the VirtualBox GUI
-    vb.name = "deis-controller"
+  config.vm.provider :vmware_fusion do |vb, override|
+    override.vm.box_url = "http://storage.core-os.net/coreos/amd64-usr/alpha/coreos_production_vagrant_vmware_fusion.box"
   end
 
-  config.vm.synced_folder "../../", "/vagrant"
+  config.vm.provider :virtualbox do |vb, override|
+    vb.customize ["modifyvm", :id, "--memory", "4096"]
+    # Fix docker not being able to resolve private registry in VirtualBox
+    vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
+    vb.customize ["modifyvm", :id, "--natdnsproxy1", "on"]
+  end
 
-  config.vm.provision :shell, inline: <<-SCRIPT
-    # Avahi-daemon broadcasts the machine's hostname to local DNS.
-    # Therefore 'deis-controller.local' in this case.
-    sudo apt-get install -yq avahi-daemon
-  SCRIPT
+  # plugin conflict
+  if Vagrant.has_plugin?("vagrant-vbguest") then
+    config.vbguest.auto_update = false
+  end
 
-end
+  (1..DEIS_NUM_INSTANCES).each do |i|
+    config.vm.define vm_name = "deis-#{i}" do |config|
+      config.vm.hostname = vm_name
+
+      ip = "172.17.8.#{i+99}"
+      config.vm.network :private_network, ip: ip
+
+      # Enable NFS for sharing the host machine into the coreos-vagrant VM.
+      config.vm.synced_folder "../..", "/home/core/share", id: "core", :nfs => true, :mount_options => ['nolock,vers=3,udp']
 
-# If you want to do some funky custom stuff to your box, but don't want those things tracked by git,
-# add a Vagrantfile.local and it will be included. You can use the exact same syntax as above. For
-# example if you're low on RAM you can boot the VM with less RAM. Note that 2GB is recommended
-# for installation, but you may be able to get away with 1GB once everything is installed.
-load "Vagrantfile.local" if File.exists? "Vagrantfile.local"
+      # workaround missing /etc/hosts
+      config.vm.provision :shell, :inline => "echo #{ip} #{vm_name} > /etc/hosts", :privileged => true
+
+      # workaround missing /etc/environment
+      config.vm.provision :shell, :inline => "touch /etc/environment", :privileged => true
+
+      # disable update-engine to prevent reboots
+      config.vm.provision :shell, :inline => "systemctl disable update-engine && systemctl mask update-engine", :privileged => true
+
+      # user-data bootstrapping
+      config.vm.provision :file, :source => "../coreos/user-data", :destination => "/tmp/user-data"
+      config.vm.provision :shell, :inline => "mkdir -p /var/lib/coreos-vagrant && mv /tmp/user-data /var/lib/coreos-vagrant", :privileged => true
+    end
+  end
+
+end