fix(*): use goamz instead of the official AWS lib

Author: Aaron Schlesinger

glide.lock

@@ -1,5 +1,9 @@
 package: github.com/deis/builder
-flatten: true
+ignore:
+- appengine
+- appengine/memcache
+- appengine/datastore
+- appengine/user
 import:
 - package: github.com/kelseyhightower/envconfig
 - package: github.com/coreos/go-etcd
@@ -27,13 +31,4 @@ import:
   - /time
 - package: github.com/codegangsta/cli
   version: a65b733b303f0055f8d324d805f393cd3e7a7904
-- package: github.com/aws/aws-sdk-go
-  version: 87b1e60a50b09e4812dee560b33a238f67305804
-  subpackages:
-  - /aws
-  - /aws/session
-ignore:
-  - appengine
-  - appengine/memcache
-  - appengine/datastore
-  - appengine/user
+- package: github.com/mitchellh/goamz

glide.yaml

@@ -10,11 +10,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg"
 	"github.com/deis/builder/pkg/gitreceive/log"
 	"github.com/deis/builder/pkg/gitreceive/storage"
+	"github.com/mitchellh/goamz/s3"
 	"github.com/pborman/uuid"
 	"gopkg.in/yaml.v2"
 )
@@ -73,11 +72,11 @@ func build(conf *Config, s3Client *s3.S3, builderKey, gitSha string) error {
 	tmpDir := os.TempDir()
 
 	tarObjKey := fmt.Sprintf("home/%s/tar", slugName)
-	tarURL := fmt.Sprintf("%s/git/%s", *s3Client.Config.Endpoint, tarObjKey)
+	tarURL := fmt.Sprintf("%s/git/%s", s3Client.S3Endpoint, tarObjKey)
 
 	// this is where workflow tells slugrunner to download the slug from, so we have to tell slugbuilder to upload it to here
 	pushObjKey := fmt.Sprintf("home/%s/push", fmt.Sprintf("%s:git-%s", appName, gitSha))
-	pushURL := fmt.Sprintf("%s/%s", *s3Client.Config.Endpoint, pushObjKey)
+	pushURL := fmt.Sprintf("%s/%s", s3Client.S3Endpoint, pushObjKey)
 
 	// Get the application config from the controller, so we can check for a custom buildpack URL
 	appConf, err := getAppConfig(conf, builderKey, conf.Username, appName)
@@ -173,15 +172,15 @@ func build(conf *Config, s3Client *s3.S3, builderKey, gitSha string) error {
 	}
 
 	bucketName := "git"
-	if _, err := s3Client.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(bucketName)}); err != nil {
+	if err := storage.CreateBucket(s3Client, bucketName); err != nil {
 		log.Warn("create bucket error: %+v", err)
 	}
 
 	appTgzReader, err := os.Open(absAppTgz)
 	if err != nil {
 		return fmt.Errorf("opening %s for read (%s)", appTgz, err)
 	}
-	if err := storage.Upload(s3Client, bucketName, tarObjKey, appTgzReader); err != nil {
+	if err := storage.UploadObject(s3Client, bucketName, tarObjKey, appTgzReader); err != nil {
 		return fmt.Errorf("uploading %s to %s/%s (%s)", absAppTgz, bucketName, tarObjKey, err)
 	}
 
@@ -235,7 +234,11 @@ func build(conf *Config, s3Client *s3.S3, builderKey, gitSha string) error {
 	// poll the s3 server to ensure the slug exists
 	// TODO: time out looking
 	for {
-		if storage.ObjectExists(s3Client, bucketName, pushObjKey) {
+		exists, err := storage.ObjectExists(s3Client, bucketName, pushObjKey)
+		if err != nil {
+			return fmt.Errorf("Checking if object %s/%s exists (%s)", bucketName, pushObjKey, err)
+		}
+		if exists {
 			break
 		}
 	}

pkg/gitreceive/build.go

@@ -6,7 +6,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/mitchellh/goamz/aws"
 )
 
 const (
@@ -17,24 +17,28 @@ const (
 var (
 	errMissingKey    = fmt.Errorf("missing %s", accessKeyIDFile)
 	errMissingSecret = fmt.Errorf("missing %s", accessSecretKeyFile)
+	emptyAuth        = &aws.Auth{}
 )
 
 // getCreds gets storage credentials from accessKeyIDFile and accessSecretKeyFile.
-// if either a key exists but not a secret or vice-versa, returns an error.
-// if neither exists, returns credentials.AnonymousCredentials
-func getCreds() (*credentials.Credentials, error) {
+// if a key exists but not a secret, or vice-versa, returns an error.
+// if both don't exist returns emptyAuth.
+// otherwise returns a valid auth
+func getAuth() (*aws.Auth, error) {
 	accessKeyIDBytes, accessKeyErr := ioutil.ReadFile(accessKeyIDFile)
 	accessSecretKeyBytes, accessSecretKeyErr := ioutil.ReadFile(accessSecretKeyFile)
 	if accessKeyErr == os.ErrNotExist && accessSecretKeyErr == os.ErrNotExist {
-		return credentials.AnonymousCredentials, nil
+		return emptyAuth, nil
 	}
 	if accessKeyErr != nil && accessSecretKeyErr == nil {
 		return nil, errMissingKey
 	}
 	if accessKeyErr == nil && accessSecretKeyErr != nil {
 		return nil, errMissingSecret
 	}
-	accessKeyID := strings.TrimSpace(string(accessKeyIDBytes))
-	accessSecretKey := strings.TrimSpace(string(accessSecretKeyBytes))
-	return credentials.NewStaticCredentials(accessKeyID, accessSecretKey, ""), nil
+
+	return &aws.Auth{
+		AccessKey: strings.TrimSpace(string(accessKeyIDBytes)),
+		SecretKey: strings.TrimSpace(string(accessSecretKeyBytes)),
+	}, nil
 }

pkg/gitreceive/storage/creds.go pkg/gitreceive/storage/auth.gopkg/gitreceive/storage/creds.go renamed to pkg/gitreceive/storage/auth.go

@@ -1,13 +1,26 @@
 package storage
 
 import (
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mitchellh/goamz/s3"
 )
 
-func CreateBucket(s3Client *s3.S3, bucketName string) error {
-	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
-		Bucket: aws.String(bucketName),
-	})
-	return err
+const (
+	ACLPublicRead = s3.ACL("public-read")
+)
+
+func BucketExists(svc *s3.S3, bucketName string) (bool, error) {
+	buckets, err := svc.ListBuckets()
+	if err != nil {
+		return false, err
+	}
+	for _, bucket := range buckets.Buckets {
+		if bucketName == bucket.Name {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func CreateBucket(svc *s3.S3, bucketName string) error {
+	return svc.Bucket(bucketName).PutBucket(s3.ACL("public-read"))
 }

pkg/gitreceive/storage/bucket.go

@@ -1,24 +1,25 @@
 package storage
 
 import (
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/mitchellh/goamz/aws"
+	"github.com/mitchellh/goamz/s3"
 )
 
-func GetClient(region string) (*s3.S3, error) {
-	awsCfg := new(aws.Config)
-	awsCfg.Region = aws.String(region)
-	endpt, err := getEndpoint()
+func GetClient(regionStr string) (*s3.S3, error) {
+	auth, err := getAuth()
 	if err != nil {
 		return nil, err
 	}
-	awsCfg.Endpoint = aws.String(endpt)
-	creds, err := getCreds()
+
+	endpoint, err := getEndpoint()
 	if err != nil {
 		return nil, err
 	}
-	awsCfg.Credentials = creds
-	svc := s3.New(session.New(awsCfg), &aws.Config{})
-	return svc, nil
+
+	region := aws.Region{
+		Name:       regionStr,
+		S3Endpoint: endpoint,
+	}
+
+	return s3.New(*auth, region), nil
 }

pkg/gitreceive/storage/client.go

@@ -0,0 +1,21 @@
+package storage
+
+import (
+	"io"
+	"net/http"
+
+	"github.com/mitchellh/goamz/s3"
+)
+
+func ObjectExists(svc *s3.S3, bucket, objName string) (bool, error) {
+	resp, err := svc.Bucket(bucket).Head(objName)
+	if err != nil {
+		return false, err
+	}
+	return resp.StatusCode == http.StatusOK, nil
+}
+
+func UploadObject(svc *s3.S3, bucketName, objKey string, reader io.Reader) error {
+	// see https://godoc.org/github.com/aws/aws-sdk-go/service/s3#example-S3-PutObject
+	return nil
+}