Merge pull request #136 from aledbf/add-timeouts

feat(builder): make the builder pod timeout configurable

Author: aledbf

boot.go

@@ -59,6 +59,8 @@ func main() {
 					pkglog.Err("Error getting config for %s [%s]", gitReceiveConfAppName, err)
 					os.Exit(1)
 				}
+				cnf.CheckDurations()
+
 				if err := gitreceive.Run(cnf); err != nil {
 					pkglog.Err("running git receive hook [%s]", err)
 					os.Exit(1)

pkg/gitreceive/build.go

@@ -10,7 +10,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg"
@@ -21,6 +20,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/util/wait"
 )
 
 // repoCmd returns exec.Command(first, others...) with its current working directory repoDir
@@ -168,9 +168,7 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, builderKey,
 		return fmt.Errorf("creating builder pod (%s)", err)
 	}
 
-	timeout := time.Duration(5 * time.Minute)
-	tick := time.Duration(500 * time.Millisecond)
-	if err := waitForPod(kubeClient, newPod.Namespace, newPod.Name, tick, timeout); err != nil {
+	if err := waitForPod(kubeClient, newPod.Namespace, newPod.Name, conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
 		return fmt.Errorf("watching events for builder pod startup (%s)", err)
 	}
 
@@ -192,15 +190,16 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, builderKey,
 	log.Debug("size of logs streamed %v", size)
 
 	// poll the s3 server to ensure the slug exists
-	// TODO: time out looking
-	for {
+	err = wait.PollImmediate(conf.ObjectStorageTickDuration(), conf.ObjectStorageWaitDuration(), func() (bool, error) {
 		exists, err := storage.ObjectExists(s3Client, bucketName, slugBuilderInfo.PushKey())
 		if err != nil {
-			return fmt.Errorf("Checking if object %s/%s exists (%s)", bucketName, slugBuilderInfo.PushKey(), err)
-		}
-		if exists {
-			break
+			return false, fmt.Errorf("Checking if object %s/%s exists (%s)", bucketName, slugBuilderInfo.PushKey(), err)
 		}
+		return exists, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("Timed out waiting for object in storage. Aborting build...")
 	}
 
 	log.Info("Build complete.")

pkg/gitreceive/config.go

@@ -2,6 +2,12 @@ package gitreceive
 
 import (
 	"strings"
+	"time"
+)
+
+const (
+	builderPodTick    = 100
+	objectStorageTick = 500
 )
 
 type Config struct {
@@ -11,15 +17,19 @@ type Config struct {
 	RegistryHost string `envconfig:"DEIS_REGISTRY_SERVICE_HOST" required:"true"`
 	RegistryPort string `envconfig:"DEIS_REGISTRY_SERVICE_PORT" required:"true"`
 
-	GitHome            string `envconfig:"GIT_HOME" required:"true"`
-	SSHConnection      string `envconfig:"SSH_CONNECTION" required:"true"`
-	SSHOriginalCommand string `envconfig:"SSH_ORIGINAL_COMMAND" required:"true"`
-	Repository         string `envconfig:"REPOSITORY" required:"true"`
-	Username           string `envconfig:"USERNAME" required:"true"`
-	Fingerprint        string `envconfig:"FINGERPRINT" required:"true"`
-	PodNamespace       string `envconfig:"POD_NAMESPACE" required:"true"`
-	StorageRegion      string `envconfig:"STORAGE_REGION" default:"us-east-1"`
-	Debug              bool   `envconfig:"DEBUG" default:"false"`
+	GitHome                       string `envconfig:"GIT_HOME" required:"true"`
+	SSHConnection                 string `envconfig:"SSH_CONNECTION" required:"true"`
+	SSHOriginalCommand            string `envconfig:"SSH_ORIGINAL_COMMAND" required:"true"`
+	Repository                    string `envconfig:"REPOSITORY" required:"true"`
+	Username                      string `envconfig:"USERNAME" required:"true"`
+	Fingerprint                   string `envconfig:"FINGERPRINT" required:"true"`
+	PodNamespace                  string `envconfig:"POD_NAMESPACE" required:"true"`
+	StorageRegion                 string `envconfig:"STORAGE_REGION" default:"us-east-1"`
+	Debug                         bool   `envconfig:"DEBUG" default:"false"`
+	BuilderPodTickDurationMSec    int    `envconfig:"BUILDER_POD_TICK_DURATION" default:"100"`
+	BuilderPodWaitDurationMSec    int    `envconfig:"BUILDER_POD_WAIT_DURATION" default:"300000"` // 5 minutes
+	ObjectStorageTickDurationMSec int    `envconfing:"OBJECT_STORAGE_TICK_DURATION" default:"500"`
+	ObjectStorageWaitDurationMSec int    `envconfig:"OBJECT_STORAGE_WAIT_DURATION" default:"300000"` // 5 minutes
 }
 
 func (c Config) App() string {
@@ -29,3 +39,45 @@ func (c Config) App() string {
 	}
 	return c.Repository[0:li]
 }
+
+// BuilderPodTickDuration returns the size of the interval used to check for
+// the end of the execution of a Pod building an application
+func (c Config) BuilderPodTickDuration() time.Duration {
+	return time.Duration(time.Duration(c.BuilderPodTickDurationMSec) * time.Millisecond)
+}
+
+// BuilderPodWaitDuration returns the maximum time to wait for the end
+// of the execution of a Pod building an application
+func (c Config) BuilderPodWaitDuration() time.Duration {
+	return time.Duration(time.Duration(c.BuilderPodWaitDurationMSec) * time.Millisecond)
+}
+
+// ObjectStorageTickDuration returns the size of the interval used to check for
+// the end of an operation that involves the object storage
+func (c Config) ObjectStorageTickDuration() time.Duration {
+	return time.Duration(time.Duration(c.ObjectStorageTickDurationMSec) * time.Millisecond)
+}
+
+// ObjectStorageWaitDuration returns the maximum time to wait for the end of an
+// operation that involves the object storage
+func (c Config) ObjectStorageWaitDuration() time.Duration {
+	return time.Duration(time.Duration(c.ObjectStorageWaitDurationMSec) * time.Millisecond)
+}
+
+// CheckDurations checks if ticks for builder and object storage are not bigger
+// than the maximum duration. In case of this it will set the tick to the default
+func (c *Config) CheckDurations() {
+	if c.BuilderPodTickDurationMSec >= c.BuilderPodWaitDurationMSec {
+		c.BuilderPodTickDurationMSec = builderPodTick
+	}
+	if c.BuilderPodTickDurationMSec < builderPodTick {
+		c.BuilderPodTickDurationMSec = builderPodTick
+	}
+
+	if c.ObjectStorageTickDurationMSec >= c.ObjectStorageWaitDurationMSec {
+		c.ObjectStorageTickDurationMSec = objectStorageTick
+	}
+	if c.ObjectStorageTickDurationMSec < objectStorageTick {
+		c.ObjectStorageTickDurationMSec = objectStorageTick
+	}
+}

pkg/gitreceive/config_test.go

@@ -0,0 +1,46 @@
+package gitreceive
+
+import (
+	"testing"
+)
+
+type checkCase struct {
+	podTick     int
+	podWait     int
+	storageTick int
+	storageWait int
+}
+
+func TestCheckDurations(t *testing.T) {
+	cases := map[checkCase]checkCase{
+		checkCase{100, 300000, 500, 300000}:    checkCase{100, 300000, 500, 300000},
+		checkCase{0, 300000, 500, 300000}:      checkCase{100, 300000, 500, 300000},
+		checkCase{100, 300000, 0, 300000}:      checkCase{100, 300000, 500, 300000},
+		checkCase{300000, 300000, 500, 300000}: checkCase{100, 300000, 500, 300000},
+		checkCase{100, 300000, 300000, 300000}: checkCase{100, 300000, 500, 300000},
+	}
+
+	var cnf Config
+	for tCase, eCase := range cases {
+		cnf = Config{
+			BuilderPodTickDurationMSec:    tCase.podTick,
+			BuilderPodWaitDurationMSec:    tCase.podWait,
+			ObjectStorageTickDurationMSec: tCase.storageTick,
+			ObjectStorageWaitDurationMSec: tCase.storageWait,
+		}
+		cnf.CheckDurations()
+
+		if cnf.BuilderPodTickDurationMSec != eCase.podTick {
+			t.Fatalf("expected %v but %v was returned (%v)", eCase.podTick, cnf.BuilderPodTickDurationMSec, tCase)
+		}
+		if cnf.BuilderPodWaitDurationMSec != eCase.podWait {
+			t.Fatalf("expected %v but %v was returned (%v)", eCase.podWait, cnf.BuilderPodWaitDurationMSec, tCase)
+		}
+		if cnf.ObjectStorageTickDurationMSec != eCase.storageTick {
+			t.Fatalf("expected %v but %v was returned (%v)", eCase.storageTick, cnf.ObjectStorageTickDurationMSec, tCase)
+		}
+		if cnf.ObjectStorageWaitDurationMSec != eCase.storageWait {
+			t.Fatalf("expected %v but %v was returned (%v)", eCase.storageWait, cnf.ObjectStorageWaitDurationMSec, tCase)
+		}
+	}
+}