refactor(tests): update test expectations and get suite passing

Author: mboersma

tests/apps_test.go

@@ -4,6 +4,7 @@ package tests
 
 import (
 	"testing"
+	"time"
 
 	"github.com/deis/deis/tests/integration-utils"
 	"github.com/deis/deis/tests/utils"
@@ -12,7 +13,6 @@ import (
 func appsSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
 	cfg.AppName = "appssample"
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")
 	cmd = itutils.GetCommand("git", "clone")
@@ -26,7 +26,7 @@ func appsCreateTest(t *testing.T, params *itutils.DeisTestConfig) {
 		t.Fatalf("Failed:\n%v", err)
 	}
 	itutils.Execute(t, cmd, params, false, "")
-	itutils.Execute(t, cmd, params, true, "Deis remote already exists")
+	itutils.Execute(t, cmd, params, true, "App with this Id already exists")
 
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatalf("Failed:\n%v", err)
@@ -43,7 +43,7 @@ func appsRunTest(t *testing.T, params *itutils.DeisTestConfig) {
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatalf("Failed:\n%v", err)
 	}
-	itutils.Execute(t, cmd, params, true, "Could not find deis remote in `git remote -v`")
+	itutils.Execute(t, cmd, params, true, "Not found")
 }
 
 func appsDestroyTest(t *testing.T, params *itutils.DeisTestConfig) {
@@ -73,6 +73,9 @@ func appsLogsTest(t *testing.T, params *itutils.DeisTestConfig) {
 		t.Fatalf("Failed:\n%v", err)
 	}
 	itutils.Execute(t, cmd1, params, false, "")
+	// TODO: nginx needs a few seconds to wake up here--fixme!
+	time.Sleep(5000 * time.Millisecond)
+	itutils.Curl(t, params)
 	itutils.Execute(t, cmd, params, false, "")
 	if err := utils.Chdir(".."); err != nil {
 		t.Fatalf("Failed:\n%v", err)

tests/builds_test.go

@@ -16,7 +16,6 @@ import (
 
 func buildSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "buildsample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")

tests/config_test.go

@@ -12,7 +12,6 @@ import (
 
 func configSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "configsample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")

tests/integration-utils/itutils.go

@@ -21,7 +21,8 @@ import (
 // Deis points to the CLI used to run tests.
 var Deis = "deis "
 
-// DeisTestConfig holds paramters needed to run tests.
+// DeisTestConfig allows tests to be repeated against different
+// targets, with different example apps, using specific credentials, and so on.
 type DeisTestConfig struct {
 	AuthKey     string
 	Hosts       string
@@ -39,9 +40,12 @@ type DeisTestConfig struct {
 	AppUser     string
 }
 
+// randomApp is used for the test run if DEIS_TEST_APP isn't set
+var randomApp = GetRandomApp()
+
 // GetGlobalConfig returns a test configuration object.
 func GetGlobalConfig() *DeisTestConfig {
-	authKey := os.Getenv("AUTH_KEY")
+	authKey := os.Getenv("DEIS_TEST_AUTH_KEY")
 	if authKey == "" {
 		authKey = "deis"
 	}
@@ -59,7 +63,7 @@ func GetGlobalConfig() *DeisTestConfig {
 	}
 	exampleApp := os.Getenv("DEIS_TEST_APP")
 	if exampleApp == "" {
-		exampleApp = "example-go"
+		exampleApp = randomApp
 	}
 	var envCfg = DeisTestConfig{
 		AuthKey:     authKey,
@@ -143,9 +147,7 @@ func CheckList(t *testing.T, params interface{}, cmd, contain string, notflag bo
 		cmdl = exec.Command("sh", "-c", Deis+cmdString)
 	}
 	if stdout, _, err := utils.RunCommandWithStdoutStderr(cmdl); err == nil {
-		if strings.Contains(stdout.String(), contain) != notflag {
-			fmt.Println("Command Executed perfectly")
-		} else {
+		if strings.Contains(stdout.String(), contain) == notflag {
 			t.Fatal(err)
 		}
 	} else {
@@ -178,13 +180,13 @@ func Execute(t *testing.T, cmd string, params interface{}, failFlag bool, expect
 	case true:
 		if stdout, stderr, err := utils.RunCommandWithStdoutStderr(cmdl); err != nil {
 			if strings.Contains(stdout.String(), expect) || strings.Contains(stderr.String(), expect) {
-				fmt.Println("Test Failed Expected behavior")
+				fmt.Println("(Error expected...ok)")
 			} else {
 				t.Fatalf("Failed:\n%v", err)
 			}
 		} else {
 			if strings.Contains(stdout.String(), expect) || strings.Contains(stderr.String(), expect) {
-				fmt.Println("expected" + expect)
+				fmt.Println("(Error expected...ok)" + expect)
 			} else {
 				t.Fatalf("Failed:\n%v", err)
 			}

tests/keys_test.go

@@ -3,12 +3,13 @@
 package tests
 
 import (
-	_ "fmt"
 	"testing"
 
 	"github.com/deis/deis/tests/integration-utils"
 )
 
+// Requires a ~/.ssh/deis-testkey to be set up:
+// $ ssh-keygen -q -t rsa -f ~/.ssh/deiskey -N '' -C deiskey
 func keysSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
 	cmd := itutils.GetCommand("auth", "login")
@@ -20,7 +21,7 @@ func keysAddTest(t *testing.T, params *itutils.DeisTestConfig) {
 	cmd := itutils.GetCommand("keys", "add")
 	params.AuthKey = "deiskey"
 	itutils.Execute(t, cmd, params, false, "")
-	itutils.Execute(t, cmd, params, true, "Uploading deiskey to Deis...400 BAD REQUEST")
+	itutils.Execute(t, cmd, params, true, "SSH Key with this Public already exists")
 }
 
 func keysListTest(t *testing.T, params *itutils.DeisTestConfig, notflag bool) {

tests/perms_test.go

@@ -12,7 +12,6 @@ import (
 
 func permsSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "permssample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")

tests/ps_test.go

@@ -4,6 +4,7 @@ package tests
 
 import (
 	_ "fmt"
+	"strings"
 	"testing"
 
 	"github.com/deis/deis/tests/integration-utils"
@@ -12,7 +13,6 @@ import (
 
 func psSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "pssample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")
@@ -34,11 +34,18 @@ func psSetup(t *testing.T) *itutils.DeisTestConfig {
 
 func psListTest(t *testing.T, params *itutils.DeisTestConfig, notflag bool) {
 	cmd := itutils.GetCommand("ps", "list")
-	itutils.CheckList(t, params, cmd, "web.2 up (v2)", notflag)
+	output := "web.2 up (v2)"
+	if strings.Contains(params.ExampleApp, "dockerfile") {
+		output = strings.Replace(output, "web", "cmd", 1)
+	}
+	itutils.CheckList(t, params, cmd, output, notflag)
 }
 
 func psScaleTest(t *testing.T, params *itutils.DeisTestConfig) {
 	cmd := itutils.GetCommand("ps", "scale")
+	if strings.Contains(params.ExampleApp, "dockerfile") {
+		cmd = strings.Replace(cmd, "web", "cmd", 1)
+	}
 	itutils.Execute(t, cmd, params, false, "")
 }
 

tests/releases_test.go

@@ -12,7 +12,6 @@ import (
 
 func releasesSetup(t *testing.T) *itutils.DeisTestConfig {
 	cfg := itutils.GetGlobalConfig()
-	cfg.ExampleApp = itutils.GetRandomApp()
 	cfg.AppName = "releasessample"
 	cmd := itutils.GetCommand("auth", "login")
 	itutils.Execute(t, cmd, cfg, false, "")

tests/smoke_test.go

@@ -11,26 +11,9 @@ import (
 	"strings"
 	"testing"
 	"text/template"
-)
-
-// A Deis test configuration allows tests to be repeated against different
-// targets, with different example apps, using specific credentials, and so on.
-type deisTestConfig struct {
-	AuthKey    string
-	ExampleApp string
-	Hosts      string
-	Domain     string
-	SSHKey     string
-}
 
-// Test configuration created from environment variables (at compile time).
-var envCfg = deisTestConfig{
-	os.Getenv("AUTH_KEY"),
-	os.Getenv("DEIS_TEST_APP"),
-	os.Getenv("DEIS_TEST_HOSTS"),
-	os.Getenv("DEIS_TEST_DOMAIN"),
-	os.Getenv("DEIS_TEST_SSH_KEY"),
-}
+	"github.com/deis/deis/tests/integration-utils"
+)
 
 // A test case is a relative directory plus a command that is expected to
 // return 0 for success.
@@ -111,36 +94,21 @@ sleep 7 && curl -s http://testing.{{.Domain}} | grep -q 'Powered by Deis' || \
 `},
 }
 
-// Updates a Vagrant instance to run Deis with docker containers using the
-// current codebase, then registers a user, pushes an example app, and looks
-// for "Powered by Deis" in the HTTP response.
+// TestSmokeExampleApp updates a Vagrant instance to run Deis with docker
+// containers using the current codebase, then registers a user, pushes an
+// example app, and looks for "Powered by Deis" in the HTTP response.
 func TestSmokeExampleApp(t *testing.T) {
-	cfg := envCfg
-	if cfg.AuthKey == "" {
-		cfg.AuthKey = "~/.ssh/deis"
-	}
-	if cfg.ExampleApp == "" {
-		cfg.ExampleApp = "example-ruby-sinatra"
-	}
-	if cfg.Hosts == "" {
-		cfg.Hosts = "local.deisapp.com"
-	}
-	if cfg.Domain == "" {
-		cfg.Domain = "local.deisapp.com"
-	}
-	if cfg.SSHKey == "" {
-		cfg.SSHKey = "~/.vagrant.d/insecure_private_key"
-	}
+	cfg := itutils.GetGlobalConfig()
 
 	for _, tt := range smokeTests {
-		runTest(t, &tt, &cfg)
+		runTest(t, &tt, cfg)
 	}
 }
 
 var wd, _ = os.Getwd()
 
 // Runs a test case and logs the results.
-func runTest(t *testing.T, tt *deisTest, cfg *deisTestConfig) {
+func runTest(t *testing.T, tt *deisTest, cfg *itutils.DeisTestConfig) {
 	// Fill in the command string template from our test configuration.
 	var cmdBuf bytes.Buffer
 	tmpl := template.Must(template.New("cmd").Parse(tt.cmd))
@@ -161,10 +129,6 @@ func runTest(t *testing.T, tt *deisTest, cfg *deisTestConfig) {
 			t.Fatal(err)
 		}
 	}
-	// TODO: Go's testing package doesn't seem to allow for reporting interim
-	// progress--we have to wait until everything completes (or fails) to see
-	// anything that was written with t.Log or t.Fatal. Interim output would
-	// be extremely helpful here, as this takes a while.
 	// Execute the command and log the input and output on error.
 	fmt.Printf("%v ... ", strings.TrimSpace(cmdString))
 	cmd := exec.Command("sh", "-c", cmdString)

tests/testconfig.json

@@ -54,7 +54,7 @@
       "unset" : "config:unset jaf --app={{.AppName}}"
     },
     "git" : {
-      "clone" : "git clone https://github.com/deis/{{.ExampleApp}}.git",
+      "clone" : "if [ ! -d {{.ExampleApp}} ] ; then git clone https://github.com/deis/{{.ExampleApp}}.git ; fi",
       "remove" : "git remote remove deis",
       "push" : "git push deis master",
       "add" : "git add .",