Merge pull request #412 from jeroenvisser101/slugbuilder-cache

feat(slugbuilder-cache): Add CACHE_PATH variable

Author: kmala

pkg/gitreceive/build.go

@@ -90,7 +90,7 @@ func build(
 		}
 	}()
 
-	slugBuilderInfo := NewSlugBuilderInfo(slugName)
+	slugBuilderInfo := NewSlugBuilderInfo(appName, gitSha.Short())
 
 	client, err := controller.New()
 	if err != nil {
@@ -184,6 +184,7 @@ func build(
 			appConf.Values,
 			slugBuilderInfo.TarKey(),
 			slugBuilderInfo.PushKey(),
+			slugBuilderInfo.CacheKey(),
 			buildPackURL,
 			conf.StorageType,
 			conf.SlugBuilderImage,

pkg/gitreceive/k8s_util.go

@@ -17,6 +17,7 @@ const (
 
 	tarPath          = "TAR_PATH"
 	putPath          = "PUT_PATH"
+	cachePath        = "CACHE_PATH"
 	debugKey         = "DEIS_DEBUG"
 	objectStore      = "objectstorage-keyfile"
 	dockerSocketName = "docker-socket"
@@ -91,6 +92,7 @@ func slugbuilderPod(
 	env map[string]interface{},
 	tarKey,
 	putKey,
+	cacheKey,
 	buildpackURL,
 	storageType,
 	image string,
@@ -104,6 +106,7 @@ func slugbuilderPod(
 
 	addEnvToPod(pod, tarPath, tarKey)
 	addEnvToPod(pod, putPath, putKey)
+	addEnvToPod(pod, cachePath, cacheKey)
 	addEnvToPod(pod, builderStorage, storageType)
 
 	if buildpackURL != "" {

pkg/gitreceive/k8s_util_test.go

@@ -29,6 +29,7 @@ type slugBuildCase struct {
 	env                        map[string]interface{}
 	tarKey                     string
 	putKey                     string
+	cacheKey                   string
 	buildPack                  string
 	slugBuilderImage           string
 	slugBuilderImagePullPolicy api.PullPolicy
@@ -56,16 +57,16 @@ func TestBuildPod(t *testing.T) {
 	var pod *api.Pod
 
 	slugBuilds := []slugBuildCase{
-		{true, "test", "default", emptyEnv, "tar", "put-url", "", "", api.PullAlways, ""},
-		{true, "test", "default", emptyEnv, "tar", "put-url", "", "", api.PullAlways, ""},
-		{true, "test", "default", env, "tar", "put-url", "", "", api.PullAlways, ""},
-		{true, "test", "default", env, "tar", "put-url", "", "", api.PullAlways, ""},
-		{true, "test", "default", emptyEnv, "tar", "put-url", "buildpack", "", api.PullAlways, ""},
-		{true, "test", "default", emptyEnv, "tar", "put-url", "buildpack", "", api.PullAlways, ""},
-		{true, "test", "default", env, "tar", "put-url", "buildpack", "", api.PullAlways, ""},
-		{true, "test", "default", env, "tar", "put-url", "buildpack", "customimage", api.PullAlways, ""},
-		{true, "test", "default", env, "tar", "put-url", "buildpack", "customimage", api.PullIfNotPresent, ""},
-		{true, "test", "default", env, "tar", "put-url", "buildpack", "customimage", api.PullNever, ""},
+		{true, "test", "default", emptyEnv, "tar", "put-url", "cache-url", "", "", api.PullAlways, ""},
+		{true, "test", "default", emptyEnv, "tar", "put-url", "cache-url", "", "", api.PullAlways, ""},
+		{true, "test", "default", env, "tar", "put-url", "cache-url", "", "", api.PullAlways, ""},
+		{true, "test", "default", env, "tar", "put-url", "cache-url", "", "", api.PullAlways, ""},
+		{true, "test", "default", emptyEnv, "tar", "put-url", "cache-url", "buildpack", "", api.PullAlways, ""},
+		{true, "test", "default", emptyEnv, "tar", "put-url", "cache-url", "buildpack", "", api.PullAlways, ""},
+		{true, "test", "default", env, "tar", "put-url", "cache-url", "buildpack", "", api.PullAlways, ""},
+		{true, "test", "default", env, "tar", "put-url", "cache-url", "buildpack", "customimage", api.PullAlways, ""},
+		{true, "test", "default", env, "tar", "put-url", "cache-url", "buildpack", "customimage", api.PullIfNotPresent, ""},
+		{true, "test", "default", env, "tar", "put-url", "cache-url", "buildpack", "customimage", api.PullNever, ""},
 	}
 
 	for _, build := range slugBuilds {
@@ -76,6 +77,7 @@ func TestBuildPod(t *testing.T) {
 			build.env,
 			build.tarKey,
 			build.putKey,
+			build.cacheKey,
 			build.buildPack,
 			build.storageType,
 			build.slugBuilderImage,

pkg/gitreceive/slug_builder_info.go

@@ -11,19 +11,23 @@ const (
 // SlugBuilderInfo contains all of the object storage related information needed to pass to a
 // slug builder.
 type SlugBuilderInfo struct {
-	pushKey string
-	tarKey  string
+	pushKey  string
+	tarKey   string
+	cacheKey string
 }
 
 // NewSlugBuilderInfo creates and populates a new SlugBuilderInfo based on the given data
-func NewSlugBuilderInfo(slugName string) *SlugBuilderInfo {
+func NewSlugBuilderInfo(appName string, shortSha string) *SlugBuilderInfo {
+	slugName := fmt.Sprintf("%s:git-%s", appName, shortSha)
 	tarKey := fmt.Sprintf("home/%s/tar", slugName)
 	// this is where workflow tells slugrunner to download the slug from, so we have to tell slugbuilder to upload it to here
 	pushKey := fmt.Sprintf("home/%s/push", slugName)
+	cacheKey := fmt.Sprintf("home/%s/cache", appName)
 
 	return &SlugBuilderInfo{
-		pushKey: pushKey,
-		tarKey:  tarKey,
+		pushKey:  pushKey,
+		tarKey:   tarKey,
+		cacheKey: cacheKey,
 	}
 }
 
@@ -36,6 +40,10 @@ func (s SlugBuilderInfo) PushKey() string { return s.pushKey }
 // folder, not including the final filename.
 func (s SlugBuilderInfo) TarKey() string { return s.tarKey }
 
+// CacheKey returns the object sotrage key that the slug builder will use to store the cache in
+// it's application specific and persisted between deploys (doesn't contain git-sha)
+func (s SlugBuilderInfo) CacheKey() string { return s.cacheKey }
+
 // AbsoluteSlugObjectKey returns the PushKey plus the final filename of the slug.
 func (s SlugBuilderInfo) AbsoluteSlugObjectKey() string { return s.PushKey() + "/" + slugTGZName }
 

pkg/gitreceive/slug_builder_info_test.go

@@ -17,7 +17,7 @@ func TestPushKey(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error building git sha (%s)", err)
 	}
-	sbi := NewSlugBuilderInfo(appName + ":git-" + sha.Short())
+	sbi := NewSlugBuilderInfo(appName, sha.Short())
 	expectedPushKey := "home/" + appName + ":git-" + sha.Short() + "/push"
 	if sbi.PushKey() != expectedPushKey {
 		t.Errorf("push key %s didn't match expected %s", sbi.PushKey(), expectedPushKey)
@@ -29,17 +29,28 @@ func TestTarKey(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error building git sha (%s)", err)
 	}
-	slugName := appName + ":git-" + sha.Short()
-	sbi := NewSlugBuilderInfo(slugName)
-	expectedTarKey := "home/" + slugName + "/tar"
+	sbi := NewSlugBuilderInfo(appName, sha.Short())
+	expectedTarKey := "home/" + appName + ":git-" + sha.Short() + "/tar"
 	if sbi.TarKey() != expectedTarKey {
 		t.Errorf("tar key %s didn't match expected %s", sbi.TarKey(), expectedTarKey)
 	}
 }
 
+func TestCacheKey(t *testing.T) {
+	sha, err := git.NewSha(rawSha)
+	if err != nil {
+		t.Fatalf("error building git sha (%s)", err)
+	}
+	sbi := NewSlugBuilderInfo(appName, sha.Short())
+	expectedCacheKey := "home/" + appName + "/cache"
+	if sbi.CacheKey() != expectedCacheKey {
+		t.Errorf("tar key %s didn't match expected %s", sbi.CacheKey(), expectedCacheKey)
+	}
+}
+
 func TestAbsoluteSlugObjectKey(t *testing.T) {
 	sha, err := git.NewSha(rawSha)
 	assert.NoErr(t, err)
-	sbi := NewSlugBuilderInfo(appName + ":git-" + sha.Short())
+	sbi := NewSlugBuilderInfo(appName, sha.Short())
 	assert.Equal(t, sbi.AbsoluteSlugObjectKey(), sbi.PushKey()+"/"+slugTGZName, "absolute slug key")
 }