Merge pull request #1497 from deis/integration-test-fixes

fix(tests): integration improvements and documentation

Author: mboersma

Makefile

@@ -160,7 +160,7 @@ test-components:
 	)
 
 test-integration:
-	$(MAKE) -C tests/ test
+	$(MAKE) -C tests/ test-full
 
 test-smoke:
 	$(MAKE) -C tests/ test-smoke

tests/Makefile

@@ -1,6 +1,4 @@
-test:
-	GOPATH=$(CURDIR)/_vendor:$(GOPATH) \
-		go test -tags integration -v -run TestSmoke
+test: test-smoke
 
 test-smoke:
 	GOPATH=$(CURDIR)/_vendor:$(GOPATH) \

tests/README.md

@@ -1,20 +1,64 @@
-# Deis integration testing
+# Deis Tests
 
-This directory contains a Go package which will run integration tests on an
-existing Deis cluster.
+This directory contains a [Go](http://golang.org/) package with integration
+tests for the [Deis](http://deis.io/) open source PaaS.
 
-Repo should be properly checked out into your GOPATH
-go get github.com/deis/deis
+[![GoDoc](https://godoc.org/github.com/deis/deis/tests?status.svg)](https://godoc.org/github.com/deis/deis/tests)
 
-To run all tests:
+**NOTE**: These integration tests are targeted for use in Deis'
+[continuous integration system](http://ci.deis.io/). The tests currently assume
+they are targeting a freshly provisioned Deis cluster. **Don't** run the
+integration tests on a Deis installation with existing users; the tests will
+fail and could overwrite data.
+
+## Test Setup
+
+Check out [Deis' source code](https://github.com/deis/deis) into the `$GOPATH`:
+
+```console
+$ go get -u -v github.com/deis/deis
+$ cd $GOPATH/src/github.com/deis/deis/tests
+```
+
+Provision a Deis cluster as usual, and ensure that a matching `deis`
+command-line client is available in your `$PATH`.
+
+Create two SSH keys:
+
+```console
+$ ssh-keygen -q -t rsa -f ~/.ssh/deis -N '' -C deis
+$ ssh-keygen -q -t rsa -f ~/.ssh/deiskey -N '' -C deiskey
+```
+
+The first key `deis` is used for authentication against Deis by the `test` user
+who runs the integration tests. The second `deiskey` is used only for testing
+`deis keys:add`, `deis keys:list`, and related commands.
+
+## Test Execution
+
+Run all the integration tests:
 
 ```console
-$ go test -v -tags integration -timeout 50m ./...
+$ make test-full
 ```
 
-The test environment uses several environment variables, which can be set to customize the run:
-* `DEIS_TEST_AUTH_KEY` - SSH key used to register with the Deis controller -- will be generated if it doesn't exist (default: `~/.ssh/deis`)
-* `DEIS_TEST_KEY` - SSH key used to login to the controller machine (default: `~/.vagrant.d/insecure_private_key`)
-* `DEIS_TEST_HOSTNAME` - hostname which resolves to the controller host (default: `local.deisapp.com`)
-* `DEIS_TEST_HOSTS` - comma-separated list of IPs for nodes in the cluster -- should be internal IPs for cloud providers (default: `172.17.8.100`)
-* `DEIS_TEST_APP` - name of the Deis example app to use, which is cloned from GitHub (default: `example-ruby-sinatra`)
+Or run just the [smoke test](http://www.catb.org/jargon/html/S/smoke-test.html):
+
+```console
+$ make test-smoke
+```
+
+## Customizing Test Runs
+
+These environment variables can be set to customize the test run:
+* `DEIS_TEST_AUTH_KEY` - SSH key used to register with the Deis controller
+  (default: `~/.ssh/deis`)
+* `DEIS_TEST_SSH_KEY` - SSH key used to login to the controller machine
+  (default: `~/.vagrant.d/insecure_private_key`)
+* `DEIS_TEST_DOMAIN` - the domain to use for testing
+  (default: `local.deisapp.com`)
+* `DEIS_TEST_HOSTS` - comma-separated list of IPs for nodes in the cluster,
+  should be internal IPs for cloud providers (default: `172.17.8.100`)
+* `DEIS_TEST_APP` - name of the
+  [Deis example app](https://github.com/deis?query=example-) to use, which is
+  cloned from GitHub (default: random)

tests/_vendor/src/github.com/dotcloud/docker/VERSION

@@ -1 +1 @@
-1.1.1
+1.1.2

tests/apps_test.go

@@ -4,6 +4,7 @@ package tests
 
 import (
 	"testing"
+	"time"
 
 	"github.com/deis/deis/tests/integration-utils"
 	"github.com/deis/deis/tests/utils"
@@ -12,7 +13,6 @@ import (
 func appsSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
 	cfg.AppName = "appssample"
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")
 	cmd = itutils.GetCommand("git", "clone")
@@ -26,7 +26,7 @@ func appsCreateTest(t *testing.T, params *itutils.DeisTestConfig) {
 		t.Fatalf("Failed:\n%v", err)
 	}
 	itutils.Execute(t, cmd, params, false, "")
-	itutils.Execute(t, cmd, params, true, "Deis remote already exists")
+	itutils.Execute(t, cmd, params, true, "App with this Id already exists")
 
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatalf("Failed:\n%v", err)
@@ -43,7 +43,7 @@ func appsRunTest(t *testing.T, params *itutils.DeisTestConfig) {
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatalf("Failed:\n%v", err)
 	}
-	itutils.Execute(t, cmd, params, true, "Could not find deis remote in `git remote -v`")
+	itutils.Execute(t, cmd, params, true, "Not found")
 }
 
 func appsDestroyTest(t *testing.T, params *itutils.DeisTestConfig) {
@@ -73,6 +73,9 @@ func appsLogsTest(t *testing.T, params *itutils.DeisTestConfig) {
 		t.Fatalf("Failed:\n%v", err)
 	}
 	itutils.Execute(t, cmd1, params, false, "")
+	// TODO: nginx needs a few seconds to wake up here--fixme!
+	time.Sleep(5000 * time.Millisecond)
+	itutils.Curl(t, params)
 	itutils.Execute(t, cmd, params, false, "")
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatalf("Failed:\n%v", err)

tests/builds_test.go

@@ -16,7 +16,6 @@ import (
 
 func buildSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "buildsample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")
@@ -58,8 +57,8 @@ func buildsListTest(t *testing.T, params *itutils.DeisTestConfig) {
 	if stdout, _, err := utils.RunCommandWithStdoutStderr(cmdl); err != nil {
 		t.Fatalf("Failed:\n%v", err)
 	} else {
-		ImageId := strings.Split(stdout.String(), "\n")[2]
-		params.ImageId = strings.Fields(ImageId)[0]
+		ImageID := strings.Split(stdout.String(), "\n")[2]
+		params.ImageID = strings.Fields(ImageID)[0]
 	}
 
 }

tests/config_test.go

@@ -12,7 +12,6 @@ import (
 
 func configSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "configsample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")

tests/dockercliutils/dockercliutils.go

@@ -1,3 +1,5 @@
+// Package dockercliutils provides helper functions for testing with Docker.
+
 package dockercliutils
 
 import (

tests/etcdutils/etcdutils.go

@@ -1,3 +1,5 @@
+// Package etcdutils helps test interactions with etcd.
+
 package etcdutils
 
 import (