feat(builder): make the builder pod timeout configurable

Author: aledbf

boot.go

@@ -17,6 +17,8 @@ import (
 const (
 	serverConfAppName     = "deis-builder-server"
 	gitReceiveConfAppName = "deis-builder-git-receive"
+	builderPodTick        = 100
+	objectStorageTick     = 500
 )
 
 func init() {
@@ -59,6 +61,15 @@ func main() {
 					pkglog.Err("Error getting config for %s [%s]", gitReceiveConfAppName, err)
 					os.Exit(1)
 				}
+
+				// check for invalid ticks duration
+				if cnf.BuilderPodTickDurationMSec >= cnf.BuilderPodWaitDurationMSec {
+					cnf.BuilderPodTickDurationMSec = builderPodTick
+				}
+				if cnf.ObjectStorageTickDurationMSec >= cnf.ObjectStorageWaitDurationMSec {
+					cnf.ObjectStorageTickDurationMSec = objectStorageTick
+				}
+
 				if err := gitreceive.Run(cnf); err != nil {
 					pkglog.Err("running git receive hook [%s]", err)
 					os.Exit(1)

pkg/gitreceive/build.go

@@ -10,7 +10,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg"
@@ -21,6 +20,7 @@ import (
 
 	"k8s.io/kubernetes/pkg/api"
 	client "k8s.io/kubernetes/pkg/client/unversioned"
+	"k8s.io/kubernetes/pkg/util/wait"
 )
 
 // repoCmd returns exec.Command(first, others...) with its current working directory repoDir
@@ -168,9 +168,7 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, builderKey,
 		return fmt.Errorf("creating builder pod (%s)", err)
 	}
 
-	timeout := time.Duration(5 * time.Minute)
-	tick := time.Duration(500 * time.Millisecond)
-	if err := waitForPod(kubeClient, newPod.Namespace, newPod.Name, tick, timeout); err != nil {
+	if err := waitForPod(kubeClient, newPod.Namespace, newPod.Name, conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
 		return fmt.Errorf("watching events for builder pod startup (%s)", err)
 	}
 
@@ -192,15 +190,16 @@ func build(conf *Config, s3Client *s3.S3, kubeClient *client.Client, builderKey,
 	log.Debug("size of logs streamed %v", size)
 
 	// poll the s3 server to ensure the slug exists
-	// TODO: time out looking
-	for {
+	err = wait.PollImmediate(conf.ObjectStorageTickDuration(), conf.ObjectStorageWaitDuration(), func() (bool, error) {
 		exists, err := storage.ObjectExists(s3Client, bucketName, slugBuilderInfo.PushKey())
 		if err != nil {
-			return fmt.Errorf("Checking if object %s/%s exists (%s)", bucketName, slugBuilderInfo.PushKey(), err)
-		}
-		if exists {
-			break
+			return false, fmt.Errorf("Checking if object %s/%s exists (%s)", bucketName, slugBuilderInfo.PushKey(), err)
 		}
+		return exists, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("Timed out waiting for object in storage. Aborting build...")
 	}
 
 	log.Info("Build complete.")

pkg/gitreceive/config.go

@@ -2,6 +2,7 @@ package gitreceive
 
 import (
 	"strings"
+	"time"
 )
 
 type Config struct {
@@ -11,15 +12,19 @@ type Config struct {
 	RegistryHost string `envconfig:"DEIS_REGISTRY_SERVICE_HOST" required:"true"`
 	RegistryPort string `envconfig:"DEIS_REGISTRY_SERVICE_PORT" required:"true"`
 
-	GitHome            string `envconfig:"GIT_HOME" required:"true"`
-	SSHConnection      string `envconfig:"SSH_CONNECTION" required:"true"`
-	SSHOriginalCommand string `envconfig:"SSH_ORIGINAL_COMMAND" required:"true"`
-	Repository         string `envconfig:"REPOSITORY" required:"true"`
-	Username           string `envconfig:"USERNAME" required:"true"`
-	Fingerprint        string `envconfig:"FINGERPRINT" required:"true"`
-	PodNamespace       string `envconfig:"POD_NAMESPACE" required:"true"`
-	StorageRegion      string `envconfig:"STORAGE_REGION" default:"us-east-1"`
-	Debug              bool   `envconfig:"DEBUG" default:"false"`
+	GitHome                       string `envconfig:"GIT_HOME" required:"true"`
+	SSHConnection                 string `envconfig:"SSH_CONNECTION" required:"true"`
+	SSHOriginalCommand            string `envconfig:"SSH_ORIGINAL_COMMAND" required:"true"`
+	Repository                    string `envconfig:"REPOSITORY" required:"true"`
+	Username                      string `envconfig:"USERNAME" required:"true"`
+	Fingerprint                   string `envconfig:"FINGERPRINT" required:"true"`
+	PodNamespace                  string `envconfig:"POD_NAMESPACE" required:"true"`
+	StorageRegion                 string `envconfig:"STORAGE_REGION" default:"us-east-1"`
+	Debug                         bool   `envconfig:"DEBUG" default:"false"`
+	BuilderPodTickDurationMSec    int    `envconfig:"BUILDER_POD_TICK_DURATION" default:"100"`
+	BuilderPodWaitDurationMSec    int    `envconfig:"BUILDER_POD_WAIT_DURATION" default:"300000"` // 5 minutes
+	ObjectStorageTickDurationMSec int    `envconfing:"OBJECT_STORAGE_TICK_DURATION" default:"500"`
+	ObjectStorageWaitDurationMSec int    `envconfig:"OBJECT_STORAGE_WAIT_DURATION" default:"300000"` // 5 minutes
 }
 
 func (c Config) App() string {
@@ -29,3 +34,27 @@ func (c Config) App() string {
 	}
 	return c.Repository[0:li]
 }
+
+// BuilderPodTickDuration returns the size of the interval used to check for
+// the end of the execution of a Pod building an application
+func (c Config) BuilderPodTickDuration() time.Duration {
+	return time.Duration(time.Duration(c.BuilderPodTickDurationMSec) * time.Millisecond)
+}
+
+// BuilderPodWaitDuration returns the maximum time to wait for the end
+// of the execution of a Pod building an application
+func (c Config) BuilderPodWaitDuration() time.Duration {
+	return time.Duration(time.Duration(c.BuilderPodWaitDurationMSec) * time.Millisecond)
+}
+
+// ObjectStorageTickDuration returns the size of the interval used to check for
+// the end of an operation that involves the object storage
+func (c Config) ObjectStorageTickDuration() time.Duration {
+	return time.Duration(time.Duration(c.ObjectStorageTickDurationMSec) * time.Millisecond)
+}
+
+// ObjectStorageWaitDuration returns the maximum time to wait for the end of an
+// operation that involves the object storage
+func (c Config) ObjectStorageWaitDuration() time.Duration {
+	return time.Duration(time.Duration(c.ObjectStorageWaitDurationMSec) * time.Millisecond)
+}