Merge branch '73-digitalocean-provider' of https://github.com/bacongobbler/deis into bacongobbler-73-digitalocean-provider

Conflicts:
	Gemfile.lock

Author: mboersma

Gemfile

@@ -6,3 +6,4 @@ gem 'berkshelf'
 
 gem 'knife-ec2'
 gem 'knife-rackspace'
+gem 'knife-digital_ocean'

Gemfile.lock

@@ -50,6 +50,11 @@ GEM
       activesupport (>= 3.2.0)
       hashie (>= 2.0.2)
       multi_json (>= 1.3.0)
+    digital_ocean (1.2.0)
+      faraday (~> 0.8.7)
+      faraday_middleware (~> 0.9.0)
+      json
+      rash
     em-winrm (0.5.4)
       eventmachine (= 1.0.0.beta.3)
       mixlib-log (>= 1.3.0)
@@ -60,7 +65,9 @@ GEM
     excon (0.25.3)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
-    ffi (1.9.0)
+    faraday_middleware (0.9.0)
+      faraday (>= 0.7.4, < 0.9)
+    ffi (1.9.1)
     fog (1.15.0)
       builder
       excon (~> 0.25.0)
@@ -85,6 +92,10 @@ GEM
     i18n (0.6.5)
     ipaddress (0.8.0)
     json (1.7.7)
+    knife-digital_ocean (0.2.0)
+      chef (>= 10.18)
+      digital_ocean (~> 1.2.0)
+      highline
     knife-ec2 (0.6.4)
       chef (>= 0.10.10)
       fog (~> 1.6)
@@ -132,6 +143,8 @@ GEM
       systemu
       yajl-ruby
     rack (1.5.2)
+    rash (0.4.0)
+      hashie (~> 2.0.0)
     rest-client (1.6.7)
       mime-types (>= 1.16)
     retryable (1.3.3)
@@ -189,5 +202,6 @@ DEPENDENCIES
   berkshelf
   chef
   foodcritic
+  knife-digital_ocean
   knife-ec2
   knife-rackspace

api/models.py

@@ -109,6 +109,7 @@ class Provider(UuidAuditedModel):
         ('mock', 'Mock Reference Provider'),
         ('rackspace', 'Rackspace Open Cloud'),
         ('static', 'Static Node'),
+        ('digitalocean', 'Digital Ocean'),
     )
 
     owner = models.ForeignKey(settings.AUTH_USER_MODEL)

client/deis.py

@@ -1657,6 +1657,14 @@ def providers_create(self, args):
                 'api_key': os.environ['RACKSPACE_API_KEY'],
                 'identity_type': os.environ.get('CLOUD_ID_TYPE', 'rackspace'),
             }
+        elif type == 'digitalocean':
+            # read creds from envvars
+            for k in ('DIGITALOCEAN_CLIENT_ID', 'DIGITALOCEAN_API_KEY'):
+                if not k in os.environ:
+                    msg = "Missing environment variable: {}".format(k)
+                    raise EnvironmentError(msg)
+            creds = {'client_id': os.environ['DIGITALOCEAN_CLIENT_ID'],
+                     'api_key': os.environ['DIGITALOCEAN_API_KEY']}
         else:
             creds = json.loads(args.get('<creds>'))
         id = args.get('<id>')  # @ReservedAssignment
@@ -1701,7 +1709,7 @@ def providers_discover(self, args):
                 else:
                     raise ResponseError(response)
         else:
-            print('No credentials discovered, did you install the EC2 Command Line tools?')
+            print('No EC2 credentials discovered. Did you install the EC2 Command Line tools?')
         if 'RACKSPACE_API_KEY' in os.environ and 'RACKSPACE_USERNAME' in os.environ:
             print("Found Rackspace credentials: {}".format(os.environ['RACKSPACE_API_KEY']))
             inp = raw_input('Import these credentials? (y/n) : ')
@@ -1721,7 +1729,27 @@ def providers_discover(self, args):
                 else:
                     raise ResponseError(response)
         else:
-            print('No Rackspace credentials discovered')
+            print('No Rackspace credentials discovered.')
+        if 'DIGITALOCEAN_API_KEY' in os.environ and 'DIGITALOCEAN_CLIENT_ID' in os.environ:
+            print("Found Digitalocean credentials: {}".format(os.environ['DIGITALOCEAN_CLIENT_ID']))
+            inp = raw_input('Import these credentials? (y/n) : ')
+            if inp.lower().strip('\n') != 'y':
+                print('Aborting.')
+            else:
+                creds = {'client_id': os.environ['DIGITALOCEAN_CLIENT_ID'],
+                         'api_key': os.environ['DIGITALOCEAN_API_KEY'],
+                }
+                body = {'creds': json.dumps(creds)}
+                sys.stdout.write('Uploading Digitalocean credentials... ')
+                sys.stdout.flush()
+                response = self._dispatch('patch', 'api/providers/digitalocean',
+                                          json.dumps(body))
+                if response.status_code == requests.codes.ok:  # @UndefinedVariable
+                    print('done')
+                else:
+                    raise ResponseError(response)
+        else:
+            print('No Digitalocean credentials discovered.')
 
     def providers_info(self, args):
         """

contrib/digitalocean/README.md

@@ -0,0 +1,34 @@
+How to Provision a Deis Controller on Digital Ocean
+===================================================
+
+Here are the steps to get started on Digital Ocean:
+
+* install [knife-digital_ocean][kdo]
+
+```
+bundle install
+```
+
+* install python requirements
+
+```
+pip install -r requirements.txt
+```
+
+* add the following to ~/.chef/knife.rb
+
+```
+knife[:digital_ocean_client_id] =   "your digital ocean client ID"
+knife[:digital_ocean_api_key] =     "your digital ocean API key"
+```
+
+* Follow the steps provided in contrib/digitalocean/prepare-digitalocean-snapshot.sh
+* Run this command to start the provisioning process
+
+```
+./contrib/digitalocean/provision-digitalocean-controller.sh
+```
+
+This script will read from your knife config file, create a new SSH key for the controller, upload the SSH key to digital ocean, and provision a Deis Controller from the snapshot created.
+
+[kdo]: https://github.com/rmoriz/knife-digital_ocean

contrib/digitalocean/prepare-digitalocean-snapshot.sh

@@ -0,0 +1,71 @@
+#!/bin/bash -ex
+
+#
+# Prepare a Deis-optimized snapshot from a vanilla Ubuntu 12.04.3 droplet.
+#
+# Instructions:
+#
+#   1. Launch a vanilla Ubuntu 12.04.3 droplet (64-bit)
+#   2. Run this script (as root!) to install the packages necessary for faster boot times
+#   3. Create a new snapshot of this droplet with the name 'deis-base'
+#   4. Create/update your Deis flavors to use your new snapshot
+#
+
+THIS_DIR=$(cd $(dirname $0); pwd) # absolute path
+CONTRIB_DIR=$(dirname $THIS_DIR)
+
+# upgrade to latest packages
+apt-get update
+apt-get upgrade -yq
+
+# install HTTPS transport support
+apt-get install -qy apt-transport-https
+
+# install docker's dependencies
+apt-get install python-software-properties -y
+
+# Add the Docker repository key to your local keychain
+# using apt-key finger you can check the fingerprint matches 36A1 D786 9245 C895 0F96 6E92 D857 6A8B A88D 21E9
+curl https://get.docker.io/gpg | apt-key add -
+
+# Add the Docker repository to your apt sources list.
+echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list
+
+# upgrade to latest packages
+apt-get update
+apt-get -qy upgrade
+
+# install required packages
+apt-get install lxc-docker curl git make python-setuptools python-pip -yq
+
+# create buildstep docker image
+git clone -b deis https://github.com/opdemand/buildstep.git
+cd buildstep
+git checkout deis
+make
+cd ..
+rm -rf buildstep
+
+# install chef 11.x deps
+apt-get install -yq ruby1.9.1 ruby1.9.1-dev make
+update-alternatives --set ruby /usr/bin/ruby1.9.1
+update-alternatives --set gem /usr/bin/gem1.9.1
+
+# clean and remove old packages
+apt-get clean
+apt-get autoremove -yq
+
+# reset cloud-init
+rm -rf /var/lib/cloud
+
+# purge SSH authorized keys
+rm -f /root/.ssh/authorized_keys
+
+# ssh host keys are automatically regenerated
+# on system boot by ubuntu cloud init
+
+# purge /var/log
+find /var/log -type f | xargs rm
+
+# flush writes to block storage
+sync

contrib/digitalocean/provision-digitalocean-controller.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+#
+# Usage: ./provision-digitalocean-controller.sh <region-id>
+#
+# Retrieve the region-id by using `knife digital_ocean region list`
+#
+
+function echo_color {
+  echo -e "\033[1m$1\033[0m"
+}
+
+THIS_DIR=$(cd $(dirname $0); pwd) # absolute path
+CONTRIB_DIR=$(dirname $THIS_DIR)
+
+echo_color "Provisioning a deis controller on Digital Ocean!"
+
+# check for Deis' general dependencies
+if ! $CONTRIB_DIR/check-deis-deps.sh; then
+  echo 'Deis is missing some dependencies.'
+  exit 1
+fi
+
+# connection details for using digital ocean's API
+client_id=$DIGITALOCEAN_CLIENT_ID
+api_key=$DIGITALOCEAN_API_KEY
+
+# Check that client ID and API key was set
+if test -z $client_id; then
+  echo "Please add your client id to ${knife_file}."
+fi
+
+if test -z $api_key; then
+  echo "Please add your api key to ${knife_file}."
+fi
+
+#################
+# chef settings #
+#################
+
+node_name=deis-controller
+run_list="recipe[deis::controller]"
+chef_version=11.4.4
+
+##########################
+# digital ocean settings #
+##########################
+
+# the name of the location we want to work with
+location_id=$1
+# The snapshot that we want to use (deis-base)
+image_id=$(knife digital_ocean image list | grep "deis-base" | awk '{print $1}')
+# the ID of the size (1GB)
+size_id=$(knife digital_ocean size list | grep "2GB" | awk '{print $1}')
+
+################
+# SSH settings #
+################
+
+key_name="deis-controller"
+ssh_key_path=~/.ssh/$key_name
+
+# create ssh keypair and store it
+if ! test -e $ssh_key_path; then
+  echo_color "Creating new SSH key: $key_name"
+  set -x
+  ssh-keygen -f $ssh_key_path -t rsa -N '' -C "deis-controller" >/dev/null
+  curl -X GET \
+    --data-urlencode "name=$node_name" \
+    --data-urlencode "ssh_pub_key=$(cat $ssh_key_path.pub)" \
+    --data-urlencode "client_id=$client_id" \
+    --data-urlencode "api_key=$api_key" \
+    https://api.digitalocean.com/ssh_keys/new
+  ssh-add $ssh_key_path
+  set +x
+  echo_color "Saved to $ssh_key_path"
+else
+  echo_color "WARNING: SSH key $ssh_key_path exists, skipping upload"
+fi
+
+# get the id of the SSH key that we just uploaded
+ssh_key_id=$(knife digital_ocean sshkey list | grep "$key_name" | awk '{print $1}')
+
+# create data bags
+knife data bag create deis-users 2>/dev/null
+knife data bag create deis-formations 2>/dev/null
+knife data bag create deis-apps 2>/dev/null
+
+# trigger digital ocean instance bootstrap
+echo_color "Provisioning $node_name with knife digital_ocean..."
+
+set -x
+knife digital_ocean droplet create \
+    --server-name $node_name \
+    --image $image_id \
+    --location $location_id \
+    --size $size_id \
+    --ssh-keys $ssh_key_id \
+    --identity-file $ssh_key_path \
+    --bootstrap \
+    --run-list $run_list
+set +x

dev_requirements.txt

@@ -9,6 +9,7 @@ django-json-field==0.5.5
 django-yamlfield==0.5
 djangorestframework==2.3.8
 git+git://github.com/surfly/gevent.git@1.0rc3#egg=gevent
+dop==0.1.4
 gunicorn==18.0
 paramiko==1.12.0
 psycopg2==2.5.1