feat(storage): implement separate storage layer

Author: Keerthan Mala

boot.go

@@ -13,7 +13,7 @@ import (
 	"github.com/deis/builder/pkg/gitreceive"
 	"github.com/deis/builder/pkg/healthsrv"
 	"github.com/deis/builder/pkg/sshd"
-	"github.com/deis/builder/pkg/storage"
+	"github.com/deis/builder/pkg/storage/driver/factory"
 	"github.com/deis/builder/pkg/sys"
 	pkglog "github.com/deis/pkg/log"
 	kcl "k8s.io/kubernetes/pkg/client/unversioned"
@@ -49,16 +49,21 @@ func main() {
 					pkglog.Err("getting config for %s [%s]", serverConfAppName, err)
 					os.Exit(1)
 				}
-				env := sys.RealEnv()
-				fs := sys.RealFS()
+
 				pushLock := sshd.NewInMemoryRepositoryLock()
 				circ := sshd.NewCircuit()
 
-				s3Client, err := storage.GetClient(cnf.HealthSrvTestStorageRegion, fs, env)
+				storageParams, err := conf.GetStorageParams()
 				if err != nil {
-					log.Printf("Error getting s3 client (%s)", err)
+					log.Printf("Error getting storage parameters (%s)", err)
 					os.Exit(1)
 				}
+				storageDriver, err := factory.Create(cnf.StorageType, storageParams)
+				if err != nil {
+					log.Printf("Error creating storage driver (%s)", err)
+					os.Exit(1)
+				}
+
 				kubeClient, err := kcl.NewInCluster()
 				if err != nil {
 					log.Printf("Error getting kubernetes client [%s]", err)
@@ -67,7 +72,7 @@ func main() {
 				log.Printf("Starting health check server on port %d", cnf.HealthSrvPort)
 				healthSrvCh := make(chan error)
 				go func() {
-					if err := healthsrv.Start(cnf.HealthSrvPort, kubeClient.Namespaces(), s3Client, circ); err != nil {
+					if err := healthsrv.Start(cnf.HealthSrvPort, kubeClient.Namespaces(), storageDriver, circ); err != nil {
 						healthSrvCh <- err
 					}
 				}()
@@ -111,8 +116,18 @@ func main() {
 				cnf.CheckDurations()
 				fs := sys.RealFS()
 				env := sys.RealEnv()
+				storageParams, err := conf.GetStorageParams()
+				if err != nil {
+					log.Printf("Error getting storage parameters (%s)", err)
+					os.Exit(1)
+				}
+				storageDriver, err := factory.Create(cnf.StorageType, storageParams)
+				if err != nil {
+					log.Printf("Error creating storage driver (%s)", err)
+					os.Exit(1)
+				}
 
-				if err := gitreceive.Run(cnf, fs, env); err != nil {
+				if err := gitreceive.Run(cnf, fs, env, storageDriver); err != nil {
 					log.Printf("Error running git receive hook [%s]", err)
 					os.Exit(1)
 				}

pkg/conf/config.go

@@ -8,9 +8,12 @@ import (
 )
 
 const (
-	builderKeyLocation = "/var/run/secrets/api/auth/builder-key"
+	builderKeyLocation  = "/var/run/secrets/api/auth/builder-key"
+	storageCredLocation = "/var/run/secrets/deis/builder/creds/"
 )
 
+type Parameters map[string]string
+
 // EnvConfig is a convenience function to process the envconfig (
 // https://github.com/kelseyhightower/envconfig) based configuration environment variables into
 // conf. Additional notes:
@@ -35,3 +38,24 @@ func GetBuilderKey() (string, error) {
 	builderKey := string(builderKeyBytes)
 	return builderKey, nil
 }
+
+func GetStorageParams() (Parameters, error) {
+	params := make(map[string]string)
+	files, err := ioutil.ReadDir(storageCredLocation)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range files {
+		data, err := ioutil.ReadFile(storageCredLocation + file.Name())
+		if err != nil {
+			return nil, err
+		}
+		if file.Name() == "key.json" {
+			params["keyfile"] = storageCredLocation + file.Name()
+		} else {
+			params[file.Name()] = string(data)
+		}
+	}
+	return params, nil
+}

pkg/gitreceive/build.go

@@ -18,6 +18,7 @@ import (
 	"github.com/deis/pkg/log"
 	"gopkg.in/yaml.v2"
 
+	storagedriver "github.com/deis/builder/pkg/storage/driver"
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 )
@@ -41,7 +42,7 @@ func run(cmd *exec.Cmd) error {
 	return cmd.Run()
 }
 
-func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs sys.FS, env sys.Env, builderKey, rawGitSha string) error {
+func build(conf *Config, storageDriver storagedriver.StorageDriver, kubeClient *client.Client, fs sys.FS, env sys.Env, builderKey, rawGitSha string) error {
 	repo := conf.Repository
 	gitSha, err := git.NewSha(rawGitSha)
 	if err != nil {
@@ -63,7 +64,7 @@ func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs
 		return fmt.Errorf("unable to create tmpdir %s (%s)", buildDir, err)
 	}
 
-	slugBuilderInfo := NewSlugBuilderInfo(s3Client.Endpoint, conf.Bucket, appName, slugName, gitSha)
+	slugBuilderInfo := storage.NewSlugBuilderInfo(slugName)
 
 	// Get the application config from the controller, so we can check for a custom buildpack URL
 	appConf, err := getAppConfig(conf, builderKey, conf.Username, appName)
@@ -100,18 +101,15 @@ func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs
 	bType := getBuildTypeForDir(tmpDir)
 	usingDockerfile := bType == buildTypeDockerfile
 
-	if err := storage.CreateBucket(s3Client, conf.Bucket); err != nil {
-		log.Warn("create bucket error: %+v", err)
-	}
-
-	appTgzReader, err := os.Open(absAppTgz)
+	appTgzdata, err := ioutil.ReadFile(absAppTgz)
 	if err != nil {
-		return fmt.Errorf("opening %s for read (%s)", appTgz, err)
+		return fmt.Errorf("error while reading file %s: (%s)", appTgz, err)
 	}
 
-	log.Debug("Uploading tar to %s/%s/%s", s3Client.Endpoint.FullURL(), conf.Bucket, slugBuilderInfo.TarKey())
-	if err := storage.UploadObject(s3Client, conf.Bucket, slugBuilderInfo.TarKey(), appTgzReader); err != nil {
-		return fmt.Errorf("uploading %s to %s/%s (%v)", absAppTgz, conf.Bucket, slugBuilderInfo.TarKey(), err)
+	//log.Debug("Uploading tar to %s/%s/%s", s3Client.Endpoint.FullURL(), conf.Bucket, slugBuilderInfo.TarKey())
+
+	if err := storageDriver.PutContent(slugBuilderInfo.TarKey(), appTgzdata); err != nil {
+		return fmt.Errorf("uploading %s to %s (%v)", absAppTgz, slugBuilderInfo.TarKey(), err)
 	}
 
 	creds := storage.CredsOK(fs)
@@ -126,7 +124,7 @@ func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs
 			buildPodName,
 			conf.PodNamespace,
 			appConf.Values,
-			slugBuilderInfo.TarURL(),
+			slugBuilderInfo.TarKey(),
 			slugName,
 			conf.StorageRegion,
 			conf.DockerBuilderImage,
@@ -139,10 +137,11 @@ func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs
 			buildPodName,
 			conf.PodNamespace,
 			appConf.Values,
-			slugBuilderInfo.TarURL(),
-			slugBuilderInfo.PushURL(),
+			slugBuilderInfo.TarKey(),
+			slugBuilderInfo.PushKey(),
 			buildPackURL,
 			conf.SlugBuilderImage,
+			conf.StorageType,
 		)
 	}
 
@@ -211,17 +210,15 @@ func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs
 	log.Debug("Done")
 
 	log.Debug(
-		"Polling the S3 server every %s for %s for the resultant slug at %s/%s",
+		"Polling the S3 server every %s for %s for the resultant slug at %s",
 		conf.ObjectStorageTickDuration(),
 		conf.ObjectStorageWaitDuration(),
-		conf.Bucket,
 		slugBuilderInfo.AbsoluteSlugObjectKey(),
 	)
 	// poll the s3 server to ensure the slug exists
 	if !usingDockerfile {
 		if err := storage.WaitForObject(
-			s3Client,
-			conf.Bucket,
+			storageDriver,
 			slugBuilderInfo.AbsoluteSlugObjectKey(),
 			conf.ObjectStorageTickDuration(),
 			conf.ObjectStorageWaitDuration(),
@@ -232,8 +229,7 @@ func build(conf *Config, s3Client *storage.Client, kubeClient *client.Client, fs
 
 	procType := pkg.ProcessType{}
 	if bType == buildTypeProcfile {
-		getter := &storage.RealObjectGetter{Client: s3Client.Client}
-		if procType, err = getProcFile(getter, tmpDir, conf.Bucket, slugBuilderInfo.AbsoluteProcfileKey()); err != nil {
+		if procType, err = getProcFile(storageDriver, tmpDir, slugBuilderInfo.AbsoluteProcfileKey()); err != nil {
 			return err
 		}
 	}
@@ -276,7 +272,7 @@ func prettyPrintJSON(data interface{}) (string, error) {
 	return string(formatted.Bytes()), nil
 }
 
-func getProcFile(getter storage.ObjectGetter, dirName, bucketName, procfileKey string) (pkg.ProcessType, error) {
+func getProcFile(storageDriver storagedriver.StorageDriver, dirName, procfileKey string) (pkg.ProcessType, error) {
 	procType := pkg.ProcessType{}
 	if _, err := os.Stat(fmt.Sprintf("%s/Procfile", dirName)); err == nil {
 		rawProcFile, err := ioutil.ReadFile(fmt.Sprintf("%s/Procfile", dirName))
@@ -289,12 +285,12 @@ func getProcFile(getter storage.ObjectGetter, dirName, bucketName, procfileKey s
 		return procType, nil
 	}
 	log.Debug("Procfile not present. Getting it from the buildpack")
-	rawProcFile, err := storage.DownloadObject(getter, bucketName, procfileKey)
+	rawProcFile, err := storageDriver.GetContent(procfileKey)
 	if err != nil {
-		return nil, fmt.Errorf("error in reading %s/%s (%s)", bucketName, procfileKey, err)
+		return nil, fmt.Errorf("error in reading %s (%s)", procfileKey, err)
 	}
 	if err := yaml.Unmarshal(rawProcFile, &procType); err != nil {
-		return nil, fmt.Errorf("procfile %s/%s is malformed (%s)", bucketName, procfileKey, err)
+		return nil, fmt.Errorf("procfile %s is malformed (%s)", procfileKey, err)
 	}
 	return procType, nil
 }

pkg/gitreceive/config.go

@@ -32,9 +32,9 @@ type Config struct {
 	BuilderPodWaitDurationMSec    int    `envconfig:"BUILDER_POD_WAIT_DURATION" default:"900000"` // 15 minutes
 	ObjectStorageTickDurationMSec int    `envconfing:"OBJECT_STORAGE_TICK_DURATION" default:"500"`
 	ObjectStorageWaitDurationMSec int    `envconfig:"OBJECT_STORAGE_WAIT_DURATION" default:"300000"` // 5 minutes
-	Bucket                        string `envconfig:"BUCKET" default:"git"`
 	SlugBuilderImage              string `envconfig:"SLUGBUILDER_IMAGE_NAME" default:"quay.io/deisci/slugbuilder:v2-beta"`
 	DockerBuilderImage            string `envconfig:"DOCKERBUILDER_IMAGE_NAME" default:"quay.io/deisci/dockerbuilder:v2-beta"`
+	StorageType                   string `envconfig:"BUILDER_STORAGE" default:"minio"`
 }
 
 // App returns the application name represented by c. The app name is the same as c.Repository

pkg/gitreceive/controller.go

@@ -168,7 +168,7 @@ func createBuildHook(
 	if !usingDockerfile {
 		ret.Dockerfile = ""
 		// need this to tell the controller what URL to give the slug runner
-		ret.Image = slugBuilderInfo.AbsoluteSlugURL()
+		ret.Image = slugBuilderInfo.AbsoluteSlugObjectKey()
 	} else {
 		ret.Dockerfile = "true"
 	}

pkg/gitreceive/k8s_util.go

@@ -18,7 +18,7 @@ const (
 	tarURLKey        = "TAR_URL"
 	putURLKey        = "put_url"
 	debugKey         = "DEBUG"
-	minioUser        = "minio-user"
+	minioUser        = "objectstorage-keyfile"
 	dockerSocketName = "docker-socket"
 	dockerSocketPath = "/var/run/docker.sock"
 )
@@ -63,14 +63,15 @@ func dockerBuilderPod(debug, withAuth bool, name, namespace string, env map[stri
 	return &pod
 }
 
-func slugbuilderPod(debug, withAuth bool, name, namespace string, env map[string]interface{}, tarURL, putURL, buildpackURL, slugBuilderImage string) *api.Pod {
+func slugbuilderPod(debug, withAuth bool, name, namespace string, env map[string]interface{}, tarKey, putURL, buildpackURL, slugBuilderImage, storageType string) *api.Pod {
 	pod := buildPod(debug, withAuth, name, namespace, env)
 
 	pod.Spec.Containers[0].Name = slugBuilderName
 	pod.Spec.Containers[0].Image = slugBuilderImage
 
-	addEnvToPod(pod, tarURLKey, tarURL)
+	addEnvToPod(pod, tarURLKey, tarKey)
 	addEnvToPod(pod, putURLKey, putURL)
+	addEnvToPod(pod, "BUILDER_STORAGE", storageType)
 
 	if buildpackURL != "" {
 		addEnvToPod(pod, "BUILDPACK_URL", buildpackURL)
@@ -112,7 +113,7 @@ func buildPod(debug, withAuth bool, name, namespace string, env map[string]inter
 		pod.Spec.Containers[0].VolumeMounts = []api.VolumeMount{
 			api.VolumeMount{
 				Name:      minioUser,
-				MountPath: "/var/run/secrets/object/store",
+				MountPath: "/var/run/secrets/deis/objectstore/creds",
 				ReadOnly:  true,
 			},
 		}

pkg/gitreceive/run.go

@@ -6,12 +6,11 @@ import (
 	"os"
 	"strings"
 
-	"github.com/deis/builder/pkg/storage"
+	builderconf "github.com/deis/builder/pkg/conf"
+	storagedriver "github.com/deis/builder/pkg/storage/driver"
 	"github.com/deis/builder/pkg/sys"
 	"github.com/deis/pkg/log"
 
-	builderconf "github.com/deis/builder/pkg/conf"
-
 	client "k8s.io/kubernetes/pkg/client/unversioned"
 )
 
@@ -25,19 +24,14 @@ func readLine(line string) (string, string, string, error) {
 
 // Run runs the git-receive hook. This func is effectively the main for the git-receive hook,
 // although it is called from the main in boot.go.
-func Run(conf *Config, fs sys.FS, env sys.Env) error {
+func Run(conf *Config, fs sys.FS, env sys.Env, storageDriver storagedriver.StorageDriver) error {
 	log.Debug("Running git hook")
 
 	builderKey, err := builderconf.GetBuilderKey()
 	if err != nil {
 		return err
 	}
 
-	s3Client, err := storage.GetClient(conf.StorageRegion, fs, env)
-	if err != nil {
-		return fmt.Errorf("configuring S3 client (%s)", err)
-	}
-
 	kubeClient, err := client.NewInCluster()
 	if err != nil {
 		return fmt.Errorf("couldn't reach the api server (%s)", err)
@@ -59,7 +53,7 @@ func Run(conf *Config, fs sys.FS, env sys.Env) error {
 		}
 		// if we're processing a receive-pack on an existing repo, run a build
 		if strings.HasPrefix(conf.SSHOriginalCommand, "git-receive-pack") {
-			if err := build(conf, s3Client, kubeClient, fs, env, builderKey, newRev); err != nil {
+			if err := build(conf, storageDriver, kubeClient, fs, env, builderKey, newRev); err != nil {
 				return err
 			}
 		}

pkg/gitreceive/slug_builder_info.go

@@ -2,9 +2,6 @@ package gitreceive
 
 import (
 	"fmt"
-
-	"github.com/deis/builder/pkg/gitreceive/git"
-	"github.com/deis/builder/pkg/storage"
 )
 
 const (
@@ -15,50 +12,32 @@ const (
 // slug builder.
 type SlugBuilderInfo struct {
 	pushKey string
-	pushURL string
 	tarKey  string
-	tarURL  string
 }
 
 // NewSlugBuilderInfo creates and populates a new SlugBuilderInfo based on the given data
-func NewSlugBuilderInfo(s3Endpoint *storage.Endpoint, bucket, appName, slugName string, gitSha *git.SHA) *SlugBuilderInfo {
+func NewSlugBuilderInfo(slugName string) *SlugBuilderInfo {
 	tarKey := fmt.Sprintf("home/%s/tar", slugName)
-	// this is where the controller tells slugrunner to download the slug from, so we have to
-	// tell slugbuilder to upload it to here
-	pushKey := fmt.Sprintf("home/%s:git-%s/push", appName, gitSha.Short())
+	// this is where workflow tells slugrunner to download the slug from, so we have to tell slugbuilder to upload it to here
+	pushKey := fmt.Sprintf("home/%s/push", slugName)
 
 	return &SlugBuilderInfo{
 		pushKey: pushKey,
-		pushURL: fmt.Sprintf("%s/%s/%s", s3Endpoint.FullURL(), bucket, pushKey),
 		tarKey:  tarKey,
-		tarURL:  fmt.Sprintf("%s/%s/%s", s3Endpoint.FullURL(), bucket, tarKey),
 	}
 }
 
 // PushKey returns the object storage key that the slug builder will store the slug in.
 // The returned value only contains the path to the folder, not including the final filename.
 func (s SlugBuilderInfo) PushKey() string { return s.pushKey }
 
-// PushURL returns the complete object storage URL that the slug builder will store the slug in.
-// The returned value only contains the URL to the folder, not including the final filename.
-func (s SlugBuilderInfo) PushURL() string { return s.pushURL }
-
 // TarKey returns the object storage key from which the slug builder will download for the tarball
 // (from which it uses to build the slug). The returned value only contains the path to the
 // folder, not including the final filename.
 func (s SlugBuilderInfo) TarKey() string { return s.tarKey }
 
-// TarURL returns the complete object storage URL that the slug builder will download the tarball
-// from. The returned value only contains the URL to the folder, not including the final filename.
-func (s SlugBuilderInfo) TarURL() string { return s.tarURL }
-
 // AbsoluteSlugObjectKey returns the PushKey plus the final filename of the slug.
 func (s SlugBuilderInfo) AbsoluteSlugObjectKey() string { return s.PushKey() + "/" + slugTGZName }
 
 // AbsoluteProcfileKey returns the PushKey plus the standard procfile name.
 func (s SlugBuilderInfo) AbsoluteProcfileKey() string { return s.PushKey() + "/Procfile" }
-
-// AbsoluteSlugURL returns the PushURL plus the final filename of the slug.
-func (s SlugBuilderInfo) AbsoluteSlugURL() string {
-	return s.PushURL() + "/" + slugTGZName
-}