Merge pull request #601 from opdemand/fix-rackspace-provider

LGTM.

Author: mboersma

Gemfile

@@ -6,5 +6,5 @@ gem 'foodcritic'
 gem "berkshelf", "~> 2.0.10"
 
 gem 'knife-ec2'
-gem 'knife-rackspace'
+gem 'knife-rackspace', '~> 0.9.0'
 gem 'knife-digital_ocean'

Gemfile.lock

@@ -25,7 +25,7 @@ GEM
     buff-config (0.4.0)
       buff-extensions (~> 0.3)
       varia_model (~> 0.1)
-    buff-extensions (1.0.0)
+    buff-extensions (0.5.0)
     buff-ignore (1.1.1)
     buff-ruby_engine (0.1.0)
     buff-shell_out (0.1.1)
@@ -66,22 +66,21 @@ GEM
       winrm (~> 1.1.2)
     erubis (2.7.0)
     eventmachine (1.0.3)
-    excon (0.25.3)
+    excon (0.31.0)
     faraday (0.8.9)
       multipart-post (~> 1.2.0)
     faraday_middleware (0.9.0)
       faraday (>= 0.7.4, < 0.9)
     ffi (1.9.3)
-    fog (1.15.0)
+    fog (1.20.0)
       builder
-      excon (~> 0.25.0)
+      excon (~> 0.31.0)
       formatador (~> 0.2.0)
       mime-types
       multi_json (~> 1.0)
       net-scp (~> 1.1)
       net-ssh (>= 2.1.3)
-      nokogiri (~> 1.5)
-      ruby-hmac
+      nokogiri (>= 1.5.11)
     foodcritic (0.2.0)
     formatador (0.2.4)
     gssapi (1.0.3)
@@ -101,13 +100,12 @@ GEM
       chef (>= 10.18)
       digital_ocean (~> 1.3.0)
       highline
-    knife-ec2 (0.6.6)
+    knife-ec2 (0.6.4)
       chef (>= 0.10.10)
-      fog (~> 1.15.0)
-      knife-windows (>= 0.5.12)
-    knife-rackspace (0.8.4)
+      fog (~> 1.6)
+    knife-rackspace (0.9.0)
       chef (>= 0.10.10)
-      fog (~> 1.12)
+      fog (~> 1.16)
       knife-windows
     knife-windows (0.5.15)
       em-winrm (~> 0.5, >= 0.5.4)
@@ -174,7 +172,6 @@ GEM
       solve (>= 0.4.4)
       varia_model (~> 0.1)
       winrm (~> 1.1.0)
-    ruby-hmac (0.4.0)
     rubyntlm (0.1.1)
     savon (0.9.5)
       akami (~> 1.0)
@@ -214,4 +211,4 @@ DEPENDENCIES
   foodcritic
   knife-digital_ocean
   knife-ec2
-  knife-rackspace
+  knife-rackspace (~> 0.9.0)

contrib/rackspace/README.md

@@ -6,54 +6,76 @@ Provision a Deis Controller on Rackspace
     ```console
     $ cd $HOME/projects/deis
     $ gem install knife-rackspace
-    Fetching: knife-rackspace-0.8.1.gem (100%)
-    Successfully installed knife-rackspace-0.8.1
+    Fetching: knife-rackspace-0.9.0.gem (100%)
+    Successfully installed knife-rackspace-0.9.0
     1 gem installed
-    Installing ri documentation for knife-rackspace-0.8.1...
-    Installing RDoc documentation for knife-rackspace-0.8.1...
+    Installing ri documentation for knife-rackspace-0.9.0...
+    Installing RDoc documentation for knife-rackspace-0.9.0...
     ```
 
-1. Export your Rackspace credentials as environment variables and edit knife.rb to read them:
+2. Export your Rackspace credentials as environment variables and edit knife.rb to read them:
 
     ```console
     $ cat <<'EOF' >> $HOME/.bash_profile
     export RACKSPACE_USERNAME=<your_rackspace_username>
     export RACKSPACE_API_KEY=<your_rackspace_api_key>
+    source $HOME/.rackspacerc
+    EOF
+    $ cat <<'EOF' > $HOME/.rackspacerc
+    export OS_AUTH_URL="https://identity.api.rackspacecloud.com/v2.0/"
+    export OS_USERNAME=$RACKSPACE_USERNAME
+    export OS_PASSWORD=$RACKSPACE_API_KEY
+    export OS_TENANT_NAME=$RACKSPACE_USERNAME
+    export OS_TENANT_ID=<your_rackspace_tenant_id>
+    export OS_REGION_NAME=<your_rackspace_region_name>
+    export OS_AUTH_SYSTEM="rackspace"
     EOF
     $ source $HOME/.bash_profile
+    $ source $HOME/.rackspacerc
     $ cat <<'EOF' >> $HOME/.chef/knife.rb
     knife[:rackspace_api_username] = "#{ENV['RACKSPACE_USERNAME']}"
     knife[:rackspace_api_key] = "#{ENV['RACKSPACE_API_KEY']}"
+    knife[:rackspace_ssh_keypair]   = "deis"
+    knife[:rackspace_region]        = #{ENV['OS_REGION_NAME']}
     EOF
     $ knife rackspace server list
     Instance ID  Name  Public IP  Private IP  Flavor  Image  State
     ```
 
-1. Prepare a new server
+3. Now you can follow the standard deis setup:
+  ```bash
+  bundle install # Installs gem files like the knife tool
+  berks install # Downloads the relevant cookbooks
+  # '--ssl-verify' is only needed when using a self-hosted Chef Server
+  # hint: you can also set that at $HOME/.berkshelf/config.json
+  berks upload [--ssl-verify=false] # Upload the cookbooks to the Chef Server
+  ```
+
+4. Prepare a new server
     1. Create a server named `deis-prepare-image` using the Ubuntu 12.04 LTS image, performance1-2, 1GB performance server
-    1. SSH in as root with the password shown
-    1. Install the 3.11 kernel with: ```apt-get update && apt-get install -yq linux-image-generic-lts-saucy linux-headers-generic-lts-saucy && reboot```
-    1. After reboot is complete, SSH back in as root and `uname -r` to confirm kernel is `3.11.0-17-generic`
-    1. Run the `prepare-controller-image.sh` script to optimize the image for fast boot times
+    2. SSH in as root with the password shown
+    3. Install the 3.11 kernel with: ```apt-get update && apt-get install -yq linux-image-generic-lts-saucy linux-headers-generic-lts-saucy && reboot```
+    4. After reboot is complete, SSH back in as root and `uname -r` to confirm kernel is `3.11.0-17-generic`
+    5. Run the `prepare-node-image.sh` script to optimize the image for fast boot times
 
         ```console
-        $ \curl -sSL https://raw2.github.com/opdemand/deis/master/contrib/rackspace/prepare-controller-image.sh | bash
+        $ ssh root@ip-address 'bash -s' < contrib/rackspace/prepare-node-image.sh
         Reading package lists... Done
         Building dependency tree
         Reading state information... Done
         ...
         ```
 
-1. Create a new image from the `deis-prepare-image` server named `deis-node-image`.
+5. Create a new image from the `deis-prepare-image` server named `deis-node-image`.
     1. In the server list in the Control Panel click the action cog for `deis-prepare-image`
-    1. Select "Create New Image" name that image `deis-node-image`
-    1. (optionally) Distribute the image to other regions
-    1. (optionally) Create/update your Deis flavors to use your new images
+    2. Select "Create New Image" name that image `deis-node-image`
+    3. (optionally) Distribute the image to other regions
+    4. (optionally) Create/update your Deis flavors to use your new images
 
-1. Make sure to add the 'deis-controller' and the '<your_username>-validator' usernames to the Chef 'admins' group.
+6. Make sure to add the `deis-controller` and the `<your_username>-validator` usernames to the Chef 'admins' group.
     * If you are using hosted Chef, you may need to use the older console to do this: <https://manage.opscode.com/groups/admins/edit>
 
-1. Back on your machine with deis cloned and the deis CLI installed, run the provisioning script to create a new Deis controller:
+7. Back on your machine with deis cloned and the deis CLI installed, run the provisioning script to create a new Deis controller:
     * Change ```<region>``` to match the region your image is in (we will add SYD and HKG as soon as performance flavors are available there):
         * dfw
         * ord
@@ -62,20 +84,28 @@ Provision a Deis Controller on Rackspace
 
         ```console
         $ cd deis
-        $ ./bundle install # if you have not already done so
+        $ bundle install # if you have not already done so
         $ ./contrib/rackspace/provision-rackspace-controller.sh <region>
         Provisioning a deis controller on Rackspace...
-        Creating new SSH key: deis-controller
-        + ssh-keygen -f /home/myuser/.ssh/deis-controller -t rsa -N '' -C deis-controller
+        Creating new SSH key: id_rsa
+        + ssh-keygen -f /home/deis/.ssh/id_rsa -t rsa -N '' -C deis
         + set +x
-        Saved to /home/myuser/.ssh/deis-controller
+        Saved to /home/deis/.ssh/id_rsa
         Created data_bag[deis-formations]
         Created data_bag[deis-apps]
-        Provisioning deis-controller with knife rackspace...
-        + knife rackspace server create -y --server-create-timeout 1200 --server-name deis-controller --image 4b7c635d-89e1-44be-a15f-2877b5a660d1 --rackspace-region <region> --flavor 4 --identity-file /home/myuser/.ssh/deis-controller --bootstrap-version 11.4.4 --node-name deis-controller --run-list 'recipe[deis::controller]'
-        Instance ID: de17ca36-f186-4cdd-8969-4be58e7108ea
-        Name: deis-controller
-        Flavor: 2GB Standard Instance
+        Provisioning deis-controller-H7WVl with knife rackspace...
+        + knife rackspace server create --bootstrap-version 11.8.2 --rackspace-region ord --image f569b831-afe5-44f5-85eb-3bf9e1d0d336 --flavor performance1-2 --rackspace-metadata '{"Name": "deis-controller-H7WVl"}' --rackspace-disk-config MANUAL --server-name deis-controller-H7WVl --node-name deis-controller-H7WVl --run-list 'recipe[deis::controller]'
+        Instance ID: cf7aeadd-4bb1-4f69-9238-7a0586a863b9
+        Name: deis-controller-H7WVl
+        Flavor: 2 GB Performance
+        Image: deis-node-image
+        Metadata: [  <Fog::Compute::RackspaceV2::Metadatum
+            key="Name",
+            value="deis-controller-H7WVl"
+          >]
+        RackConnect Wait: no
+        ServiceLevel Wait: no
+        SSH Key: deis
         ...
         ```
 

contrib/rackspace/prepare-controller-image.sh

@@ -1,13 +1,19 @@
 #!/bin/bash -ex
 
 #
-# Prepare a Deis Node image for Rackspace.
+# Prepare a Deis image for Rackspace.
 #
 # Instructions:
 #
-#   1. Launch a vanilla Ubuntu 12.04.3 droplet (64-bit)
-#   2. Run this script (as root!) to optimize the image for faster provisioning
-#   3. Create a new snapshot named "deis-node-image" from this droplet
+#   1. Create a server using the Ubuntu 12.04 LTS image,
+#      performance1-2, 2GB performance server
+#   2. SSH in as root with the password shown, then install the 3.11 kernel with:
+#      apt-get update && apt-get install -yq linux-image-generic-lts-saucy linux-headers-generic-lts-saucy && reboot
+#   3. After reboot is complete, SSH in and `uname -r` to confirm kernel is 3.11
+#   4. Run this script (as root) to optimize the image for fast boot times
+#   5. Create a new image from the server named "deis-node-image".
+#   6. Distribute the image to other regions
+#   7. Create/update your Deis flavors to use your new images
 #
 
 # Remove old kernel(s)
@@ -22,7 +28,7 @@ echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/dock
 
 # upgrade to latest packages
 apt-get update
-apt-get upgrade -yq
+apt-get dist-upgrade -yq
 
 # install required packages
 apt-get install lxc-docker-0.8.0 fail2ban curl git inotify-tools make python-setuptools python-pip -yq
@@ -32,8 +38,16 @@ while [ ! -e /var/run/docker.sock ] ; do
   inotifywait -t 2 -e create $(dirname /var/run/docker.sock)
 done
 
-# pull docker images
-docker pull deis/slugrunner:latest
+# pull current docker images
+docker pull deis/data:latest
+docker pull deis/discovery:latest
+docker pull deis/registry:latest
+docker pull deis/cache:latest
+docker pull deis/logger:latest
+docker pull deis/database:latest
+docker pull deis/server:latest
+docker pull deis/worker:latest
+docker pull deis/builder:latest
 
 # install chef 11.x deps
 apt-get install -yq ruby1.9.1 ruby1.9.1-dev make

contrib/rackspace/prepare-node-image.sh

@@ -21,16 +21,15 @@ if ! "$CONTRIB_DIR/check-deis-deps.sh"; then
   exit 1
 fi
 
-# check for knife-rackspace
-if ! knife rackspace server list > /dev/null; then
-  echo 'Please install the knife-rackspace Ruby gem and configure knife.rb.'
+if ! nova --version > /dev/null 2>&1; then
+  echo "Please install nova using 'pip install python-novaclient'."
   exit 1
 fi
 
 #################
 # chef settings #
 #################
-node_name=deis-controller
+node_name="deis-controller-$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom | head -c 5 | xargs)"
 run_list="recipe[deis::controller]"
 chef_version=11.8.2
 
@@ -57,21 +56,28 @@ fi
 ################
 # SSH settings #
 ################
-key_name=deis-controller
+key_name=id_rsa
 ssh_key_path=~/.ssh/$key_name
-ssh_user="ubuntu"  # doesn't work?
+ssh_user="root"
 
 # create ssh keypair and store it
 if ! test -e $ssh_key_path; then
   echo_color "Creating new SSH key: $key_name"
   set -x
-  ssh-keygen -f $ssh_key_path -t rsa -N '' -C "deis-controller" >/dev/null
+  ssh-keygen -f $ssh_key_path -t rsa -N '' -C "$USER" >/dev/null
   set +x
   echo_color "Saved to $ssh_key_path"
 else
   echo_color "WARNING: SSH key $ssh_key_path exists"
 fi
 
+# upload the user's SSH key to Rackspace.
+# if it fails, that means that it's already been uploaded.
+echo "uploading keypair to Rackspace, please wait"
+if ! nova keypair-add --pub-key $ssh_key_path.pub deis > /dev/null; then
+  echo "keypair already uploaded to Rackspace. Skipping."
+fi
+
 # create data bags
 knife data bag create deis-formations 2>/dev/null
 knife data bag create deis-apps 2>/dev/null
@@ -86,7 +92,6 @@ knife rackspace server create \
  --flavor $flavor \
  --rackspace-metadata "{\"Name\": \"$node_name\"}" \
  --rackspace-disk-config MANUAL \
- --identity-file $ssh_key_path \
  --server-name $node_name \
  --node-name $node_name \
  --run-list $run_list