Merge pull request #223 from arschles/gcs-compat

fix(pkg/gitreceive): replace the AWS SDK with minio-go

Author: arschles

glide.lock

@@ -11,13 +11,13 @@ import:
 - package: github.com/Masterminds/cookoo
   version: 110a04ff7dc3e7c9b86c1a2413906a16a2bb65cb
   subpackages:
-  - /log
-  - /safely
-  - /fmt
+  - log
+  - safely
+  - fmt
 - package: golang.org/x/crypto
   version: f7445b17d61953e333441674c2d11e91ae4559d3
   subpackages:
-  - /ssh
+  - ssh
 - package: gopkg.in/yaml.v2
   version: eca94c41d994ae2215d455ce578ae6e2dc6ee516
 - package: github.com/pborman/uuid
@@ -28,13 +28,8 @@ import:
   - log
 - package: github.com/codegangsta/cli
   version: a65b733b303f0055f8d324d805f393cd3e7a7904
-- package: github.com/aws/aws-sdk-go
-  version: 87b1e60a50b09e4812dee560b33a238f67305804
-  subpackages:
-  - aws
-  - aws/session
-  - service/s3
 - package: k8s.io/kubernetes
   version: ~1.1
 - package: github.com/arschles/assert
   version: 6882f85ccdc7c1822b146d1a6b0c2c48f91b5140
+- package: github.com/minio/minio-go

glide.yaml

@@ -11,7 +11,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg"
 	"github.com/deis/builder/pkg/gitreceive/git"
 	"github.com/deis/builder/pkg/gitreceive/storage"
@@ -21,7 +20,6 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
-	"k8s.io/kubernetes/pkg/util/wait"
 )
 
 // repoCmd returns exec.Command(first, others...) with its current working directory repoDir
@@ -42,7 +40,7 @@ func run(cmd *exec.Cmd) error {
 	return cmd.Run()
 }
 
-func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, fs sys.FS, env sys.Env, builderKey, rawGitSha string) error {
+func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs sys.FS, env sys.Env, builderKey, rawGitSha string) error {
 	repo := conf.Repository
 	gitSha, err := git.NewSha(rawGitSha)
 	if err != nil {
@@ -121,7 +119,7 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, fs sys.FS,
 		return fmt.Errorf("opening %s for read (%s)", appTgz, err)
 	}
 
-	log.Debug("Uploading tar to %s/%s/%s", s3Client.Endpoint, conf.Bucket, slugBuilderInfo.TarKey())
+	log.Debug("Uploading tar to %s/%s/%s", s3Client.Endpoint.FullURL(), conf.Bucket, slugBuilderInfo.TarKey())
 	if err := storage.UploadObject(s3Client, conf.Bucket, slugBuilderInfo.TarKey(), appTgzReader); err != nil {
 		return fmt.Errorf("uploading %s to %s/%s (%v)", absAppTgz, conf.Bucket, slugBuilderInfo.TarKey(), err)
 	}
@@ -193,11 +191,20 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, fs sys.FS,
 	}
 	log.Debug("size of streamed logs %v", size)
 
+	log.Debug(
+		"Waiting for the %s/%s pod to end. Checking every %s for %s",
+		newPod.Namespace,
+		newPod.Name,
+		conf.BuilderPodTickDuration(),
+		conf.BuilderPodWaitDuration(),
+	)
 	// check the state and exit code of the build pod.
 	// if the code is not 0 return error
 	if err := waitForPodEnd(kubeClient, newPod.Namespace, newPod.Name, conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
 		return fmt.Errorf("error getting builder pod status (%s)", err)
 	}
+	log.Debug("Done")
+	log.Debug("Checking for builder pod exit code")
 	buildPod, err := kubeClient.Pods(newPod.Namespace).Get(newPod.Name)
 	if err != nil {
 		return fmt.Errorf("error getting builder pod status (%s)", err)
@@ -209,37 +216,30 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, fs sys.FS,
 			return fmt.Errorf("Build pod exited with code %d, stopping build.", state.ExitCode)
 		}
 	}
-
+	log.Debug("Done")
+
+	log.Debug(
+		"Polling the S3 server every %s for %s for the resultant slug at %s/%s",
+		conf.ObjectStorageTickDuration(),
+		conf.ObjectStorageWaitDuration(),
+		conf.Bucket,
+		slugBuilderInfo.AbsoluteSlugObjectKey(),
+	)
 	// poll the s3 server to ensure the slug exists
-	err = wait.PollImmediate(conf.ObjectStorageTickDuration(), conf.ObjectStorageWaitDuration(), func() (bool, error) {
-		exists, err := storage.ObjectExists(s3Client, conf.Bucket, slugBuilderInfo.PushKey())
-		if err != nil {
-			return false, fmt.Errorf("Checking if object %s/%s exists (%s)", conf.Bucket, slugBuilderInfo.PushKey(), err)
-		}
-		return exists, nil
-	})
-
-	if err != nil {
-		return fmt.Errorf("Timed out waiting for object in storage. Aborting build...")
+	if err := storage.WaitForObject(
+		s3Client,
+		conf.Bucket,
+		slugBuilderInfo.AbsoluteSlugObjectKey(),
+		conf.ObjectStorageTickDuration(),
+		conf.ObjectStorageWaitDuration(),
+	); err != nil {
+		return fmt.Errorf("Timed out waiting for object in storage, aborting build (%s)", err)
 	}
 	log.Info("Build complete.")
 	log.Info("Launching app.")
 	log.Info("Launching...")
 
-	buildHook := &pkg.BuildHook{
-		Sha:         gitSha.Short(),
-		ReceiveUser: conf.Username,
-		ReceiveRepo: appName,
-		Image:       appName,
-		Procfile:    procType,
-	}
-	if !usingDockerfile {
-		buildHook.Dockerfile = ""
-		// need this to tell the controller what URL to give the slug runner
-		buildHook.Image = slugBuilderInfo.PushURL() + "/slug.tgz"
-	} else {
-		buildHook.Dockerfile = "true"
-	}
+	buildHook := createBuildHook(slugBuilderInfo, gitSha, conf.Username, appName, procType, usingDockerfile)
 	buildHookResp, err := publishRelease(conf, builderKey, buildHook)
 	if err != nil {
 		return fmt.Errorf("publishing release (%s)", err)

pkg/gitreceive/build.go

@@ -9,6 +9,8 @@ import (
 	"strings"
 
 	"github.com/deis/builder/pkg"
+	"github.com/deis/builder/pkg/gitreceive/git"
+	"github.com/deis/builder/pkg/gitreceive/storage"
 	"github.com/deis/pkg/log"
 )
 
@@ -148,3 +150,28 @@ func receive(conf *Config, builderKey, gitSha string) error {
 	}
 	return nil
 }
+
+func createBuildHook(
+	slugBuilderInfo *storage.SlugBuilderInfo,
+	gitSha *git.SHA,
+	username,
+	appName string,
+	procType pkg.ProcessType,
+	usingDockerfile bool,
+) *pkg.BuildHook {
+	ret := &pkg.BuildHook{
+		Sha:         gitSha.Short(),
+		ReceiveUser: username,
+		ReceiveRepo: appName,
+		Image:       appName,
+		Procfile:    procType,
+	}
+	if !usingDockerfile {
+		ret.Dockerfile = ""
+		// need this to tell the controller what URL to give the slug runner
+		ret.Image = slugBuilderInfo.AbsoluteSlugURL()
+	} else {
+		ret.Dockerfile = "true"
+	}
+	return ret
+}

pkg/gitreceive/controller.go

@@ -0,0 +1,42 @@
+package gitreceive
+
+import (
+	"testing"
+
+	"github.com/arschles/assert"
+	"github.com/deis/builder/pkg"
+	"github.com/deis/builder/pkg/gitreceive/git"
+	"github.com/deis/builder/pkg/gitreceive/storage"
+)
+
+const (
+	rawSha   = "c3b4e4ba8b7267226ff02ad07a3a2cca9c9237de"
+	bucket   = "git"
+	appName  = "myapp"
+	slugName = "myslug"
+	username = "myuser"
+)
+
+func TestCreateBuildHook(t *testing.T) {
+	procType := pkg.ProcessType(make(map[string]string))
+	sha, err := git.NewSha(rawSha)
+	assert.NoErr(t, err)
+	endpoint := &storage.Endpoint{URLStr: "s3.amazonaws.com", Secure: false}
+	slugBuilderInfo := storage.NewSlugBuilderInfo(endpoint, bucket, appName, slugName, sha)
+	hookUsingDockerfile := createBuildHook(slugBuilderInfo, sha, username, appName, procType, true)
+	assert.Equal(t, hookUsingDockerfile.Sha, sha.Short(), "git sha")
+	assert.Equal(t, hookUsingDockerfile.ReceiveUser, username, "username")
+	assert.Equal(t, hookUsingDockerfile.ReceiveRepo, appName, "username")
+	assert.Equal(t, hookUsingDockerfile.Image, appName, "image")
+	assert.Equal(t, hookUsingDockerfile.Procfile, procType, "procfile")
+	assert.Equal(t, hookUsingDockerfile.Dockerfile, "true", "dockerfile field")
+
+	hookNoDockerfile := createBuildHook(slugBuilderInfo, sha, username, appName, procType, false)
+	assert.Equal(t, hookNoDockerfile.Sha, sha.Short(), "git sha")
+	assert.Equal(t, hookNoDockerfile.ReceiveUser, username, "username")
+	assert.Equal(t, hookNoDockerfile.ReceiveRepo, appName, "username")
+	assert.Equal(t, hookNoDockerfile.Image, slugBuilderInfo.AbsoluteSlugURL(), "image")
+	assert.Equal(t, hookNoDockerfile.Procfile, procType, "procfile")
+	assert.Equal(t, hookNoDockerfile.Dockerfile, "", "dockerfile field")
+
+}

pkg/gitreceive/controller_test.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/deis/builder/pkg/sys"
 )
 
@@ -17,18 +16,27 @@ const (
 var (
 	errMissingKey    = fmt.Errorf("missing %s", accessKeyIDFile)
 	errMissingSecret = fmt.Errorf("missing %s", accessSecretKeyFile)
-	emptyAuth        = credentials.AnonymousCredentials
+	emptyCreds       = creds{}
 )
 
+type creds struct {
+	accessKeyID     string
+	accessKeySecret string
+}
+
+func (c *creds) isZero() bool {
+	return c.accessKeyID == "" && c.accessKeySecret == ""
+}
+
 // getAuth gets storage credentials from accessKeyIDFile and accessSecretKeyFile.
 // if a key exists but not a secret, or vice-versa, returns an error.
 // if both don't exist returns emptyAuth.
 // otherwise returns a valid auth
-func getAuth(fs sys.FS) (*credentials.Credentials, error) {
+func getAuth(fs sys.FS) (*creds, error) {
 	accessKeyIDBytes, accessKeyErr := fs.ReadFile(accessKeyIDFile)
 	accessSecretKeyBytes, accessSecretKeyErr := fs.ReadFile(accessSecretKeyFile)
 	if accessKeyErr == os.ErrNotExist && accessSecretKeyErr == os.ErrNotExist {
-		return emptyAuth, nil
+		return &emptyCreds, nil
 	}
 	if accessKeyErr != nil && accessSecretKeyErr == nil {
 		return nil, errMissingKey
@@ -39,18 +47,17 @@ func getAuth(fs sys.FS) (*credentials.Credentials, error) {
 
 	id := strings.TrimSpace(string(accessKeyIDBytes))
 	secret := strings.TrimSpace(string(accessSecretKeyBytes))
-	return credentials.NewStaticCredentials(id, secret, ""), nil
+	return &creds{accessKeyID: id, accessKeySecret: secret}, nil
 }
 
 // CredsOK checks if the required credentials to make a request exist
 func CredsOK(fs sys.FS) bool {
-	cred, err := getAuth(fs)
+	creds, err := getAuth(fs)
 	if err != nil {
 		return false
 	}
 
-	auth, _ := cred.Get()
-	if auth.AccessKeyID == "" && auth.SecretAccessKey == "" {
+	if creds.accessKeyID == "" && creds.accessKeySecret == "" {
 		return false
 	}