fix(pkg/gitreceive): replace the AWS SDK with minio-go

Author: Aaron Schlesinger

pkg/gitreceive/build.go

@@ -11,12 +11,12 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg"
 	"github.com/deis/builder/pkg/gitreceive/git"
 	"github.com/deis/builder/pkg/gitreceive/storage"
 	"github.com/deis/builder/pkg/sys"
 	"github.com/deis/pkg/log"
+	s3 "github.com/minio/minio-go"
 	"gopkg.in/yaml.v2"
 
 	"k8s.io/kubernetes/pkg/api"
@@ -42,7 +42,7 @@ func run(cmd *exec.Cmd) error {
 	return cmd.Run()
 }
 
-func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, fs sys.FS, env sys.Env, builderKey, rawGitSha string) error {
+func build(conf *Config, s3Client *s3.Client, kubeClient *client.Client, fs sys.FS, env sys.Env, builderKey, rawGitSha string) error {
 	repo := conf.Repository
 	gitSha, err := git.NewSha(rawGitSha)
 	if err != nil {

pkg/gitreceive/storage/auth.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/deis/builder/pkg/sys"
 )
 
@@ -20,6 +19,11 @@ var (
 	emptyAuth        = credentials.AnonymousCredentials
 )
 
+type creds struct {
+	accessKeyID     string
+	accessKeySecret string
+}
+
 // getAuth gets storage credentials from accessKeyIDFile and accessSecretKeyFile.
 // if a key exists but not a secret, or vice-versa, returns an error.
 // if both don't exist returns emptyAuth.
@@ -39,7 +43,7 @@ func getAuth(fs sys.FS) (*credentials.Credentials, error) {
 
 	id := strings.TrimSpace(string(accessKeyIDBytes))
 	secret := strings.TrimSpace(string(accessSecretKeyBytes))
-	return credentials.NewStaticCredentials(id, secret, ""), nil
+	return &creds{accessKeyID: id, accessKeySecret: secret}, nil
 }
 
 // CredsOK checks if the required credentials to make a request exist
@@ -49,8 +53,7 @@ func CredsOK(fs sys.FS) bool {
 		return false
 	}
 
-	auth, _ := cred.Get()
-	if auth.AccessKeyID == "" && auth.SecretAccessKey == "" {
+	if creds.accessKeyID == "" && creds.accessKeySecret == "" {
 		return false
 	}
 

pkg/gitreceive/storage/bucket.go

@@ -1,37 +1,29 @@
 package storage
 
 import (
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/service/s3"
+	s3 "github.com/minio/minio-go"
 )
 
 const (
 	bucketAlreadyExistsCode = "BucketAlreadyExists"
+	nonExistentBucketCode   = "NoSuchBucket"
 )
 
 var (
 	// ACLPublicRead default ACL for objects in the S3 API compatible storage
-	ACLPublicRead = aws.String("public-read")
+	ACLPublicRead = "public-read"
 )
 
 // CreateBucket creates a new bucket in the S3 API compatible storage or
 // return an error in case the bucket already exists
-func CreateBucket(svc *s3.S3, bucketName string) error {
-	_, err := svc.CreateBucket(&s3.CreateBucketInput{
-		Bucket: aws.String(bucketName),
-		ACL:    ACLPublicRead,
-	})
-
+func CreateBucket(creator BucketCreator, bucketName string) error {
+	_, err := creator.MakeBucket(bucketName, ACLPublicRead, "")
 	if err != nil {
-		if awsErr, ok := err.(awserr.Error); ok {
-			if awsErr.Code() == bucketAlreadyExistsCode {
-				return nil
-			}
+		minioErr := s3.ToErrorResponse(err)
+		if minioErr.Code == bucketAlreadyExistsCode {
+			return nil
 		}
-
 		return err
 	}
-
 	return nil
 }

pkg/gitreceive/storage/client.go

@@ -1,14 +1,12 @@
 package storage
 
 import (
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg/sys"
+	s3 "github.com/minio/minio-go"
 )
 
 // GetClient returns a S3 API compatible storage client
-func GetClient(regionStr string, fs sys.FS, env sys.Env) (*s3.S3, error) {
+func GetClient(regionStr string, fs sys.FS, env sys.Env) (*s3.Client, error) {
 	auth, err := getAuth(fs)
 	if err != nil {
 		return nil, err
@@ -19,10 +17,5 @@ func GetClient(regionStr string, fs sys.FS, env sys.Env) (*s3.S3, error) {
 		return nil, err
 	}
 
-	return s3.New(session.New(&aws.Config{
-		Credentials:      auth,
-		Region:           aws.String(regionStr),
-		Endpoint:         aws.String(endpoint),
-		S3ForcePathStyle: aws.Bool(true),
-	})), nil
+	return s3.New(endpoint, auth.accessKeyID, auth.accessKeySecret, false), nil
 }

pkg/gitreceive/storage/endpoint.go

@@ -2,6 +2,8 @@ package storage
 
 import (
 	"fmt"
+	"os"
+	"strings"
 
 	"github.com/deis/builder/pkg/sys"
 )
@@ -21,15 +23,30 @@ var (
 	)
 )
 
-func getEndpoint(env sys.Env) (string, error) {
+func stripScheme(str string) string {
+	schemes := []string{"http://", "https://"}
+	for _, scheme := range schemes {
+		if strings.HasPrefix(str, scheme) {
+			str = str[len(scheme):]
+		}
+	}
+	return str
+}
+
+type endpoint struct {
+	urlStr string
+	secure bool
+}
+
+func getEndpoint(env sys.Env) (*endpoint, error) {
 	mHost := env.Get(minioHostEnvVar)
 	mPort := env.Get(minioPortEnvVar)
 	S3EP := env.Get(outsideStorageEndpoint)
 	if S3EP != "" {
-		return S3EP, nil
+		return &endpoint{urlStr: stripScheme(S3EP), secure: true}, nil
 	} else if mHost != "" && mPort != "" {
-		return fmt.Sprintf("http://%s:%s", mHost, mPort), nil
+		return &endpoint{urlStr: fmt.Sprintf("%s:%s", mHost, mPort), secure: false}, nil
 	} else {
-		return "", errNoStorageConfig
+		return nil, errNoStorageConfig
 	}
 }

pkg/gitreceive/storage/interfaces.go

@@ -0,0 +1,17 @@
+package storage
+
+import (
+	s3 "github.com/minio/minio-go"
+)
+
+type BucketCreator interface {
+	MakeBucket(bucketName string, acl s3.BucketACL, location string) error
+}
+
+type ObjectStatter interface {
+	StatObject(bucketName, objectKey string) (s3.ObjectInfo, error)
+}
+
+type ObjectPutter interface {
+	PutObject(bucketName, objectKey string, reader io.Reader, contentType string) (int64, error)
+}

pkg/gitreceive/storage/object.go

@@ -3,27 +3,27 @@ package storage
 import (
 	"io"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/service/s3"
+	s3 "github.com/minio/minio-go"
 )
 
-func ObjectExists(svc *s3.S3, bucket, objName string) (bool, error) {
-	_, err := svc.HeadBucket(&s3.HeadBucketInput{
-		Bucket: aws.String(bucket),
-	})
+const (
+	noSuchKeyCode = "NoSuchKey"
+
+	octetStream = "application/octet-stream"
+)
+
+func ObjectExists(statter ObjectStatter, bucket, objName string) (bool, error) {
+	objInfo, err := statter.StatObject(bucketName, objKey)
 	if err != nil {
 		return false, err
 	}
+	if objInfo.Code == noSuchKeyCode || objInfo.Err != nil {
+		return false, nil
+	}
 	return true, nil
 }
 
-func UploadObject(svc *s3.S3, bucketName, objKey string, reader io.Reader) error {
-	params := &s3.PutObjectInput{
-		Body:   aws.ReadSeekCloser(reader),
-		Bucket: aws.String(bucketName),
-		Key:    aws.String(objKey),
-		ACL:    ACLPublicRead,
-	}
-	_, err := svc.PutObject(params)
+func UploadObject(putter ObjectPutter, bucketName, objKey string, reader io.Reader) error {
+	_, err := putter.PutObject(bucketName, objKey, reader, octetStream)
 	return err
 }