fix(*): don't report the server up until the SSH server is running

the builder won’t be considered up until the SSH server is actually
running

Author: Aaron Schlesinger

boot.go

@@ -48,6 +48,9 @@ func main() {
 					pkglog.Err("getting config for %s [%s]", serverConfAppName, err)
 					os.Exit(1)
 				}
+
+				serverCircuit := sshd.NewCircuit()
+
 				s3Client, err := storage.GetClient(cnf.HealthSrvTestStorageRegion)
 				if err != nil {
 					pkglog.Err("getting s3 client [%s]", err)
@@ -61,15 +64,15 @@ func main() {
 				pkglog.Info("starting health check server on port %d", cnf.HealthSrvPort)
 				healthSrvCh := make(chan error)
 				go func() {
-					if err := healthsrv.Start(cnf.HealthSrvPort, kubeClient.Namespaces(), s3Client); err != nil {
+					if err := healthsrv.Start(cnf.HealthSrvPort, kubeClient.Namespaces(), s3Client, serverCircuit); err != nil {
 						healthSrvCh <- err
 					}
 				}()
 
 				pkglog.Info("starting SSH server on %s:%d", cnf.SSHHostIP, cnf.SSHHostPort)
 				sshCh := make(chan int)
 				go func() {
-					sshCh <- pkg.Run(cnf.SSHHostIP, cnf.SSHHostPort, "boot")
+					sshCh <- pkg.RunBuilder(cnf.SSHHostIP, cnf.SSHHostPort, serverCircuit)
 				}()
 
 				select {

pkg/builder.go

@@ -5,13 +5,12 @@ package pkg
 
 import (
 	"fmt"
+	"log"
+	"os"
 
 	"github.com/Masterminds/cookoo"
 	clog "github.com/Masterminds/cookoo/log"
 	"github.com/deis/builder/pkg/sshd"
-
-	"log"
-	"os"
 )
 
 // Return codes that will be sent to the shell.
@@ -28,7 +27,7 @@ const (
 // Git.
 //
 // Run returns on of the Status* status code constants.
-func Run(sshHostIP string, sshHostPort int, cmd string) int {
+func RunBuilder(sshHostIP string, sshHostPort int, sshServerCircuit *sshd.Circuit) int {
 	reg, router, ocxt := cookoo.Cookoo()
 	log.SetFlags(0) // Time is captured elsewhere.
 
@@ -59,7 +58,7 @@ func Run(sshHostIP string, sshHostPort int, cmd string) int {
 	// Start the SSH service.
 	// TODO: We could refactor Serve to be a command, and then run this as
 	// a route.
-	if err := sshd.Serve(reg, router, cxt); err != nil {
+	if err := sshd.Serve(reg, router, sshServerCircuit, cxt); err != nil {
 		clog.Errf(cxt, "SSH server failed: %s", err)
 		return StatusLocalError
 	}

pkg/healthsrv/healthz_handler.go

@@ -8,6 +8,7 @@ import (
 
 	s3 "github.com/aws/aws-sdk-go/service/s3"
 	"github.com/deis/builder/pkg/gitreceive/log"
+	"github.com/deis/builder/pkg/sshd"
 	"k8s.io/kubernetes/pkg/fields"
 	"k8s.io/kubernetes/pkg/labels"
 )
@@ -25,12 +26,21 @@ func convertBucket(b *s3.Bucket) healthZRespBucket {
 }
 
 type healthZResp struct {
-	Namespaces []string            `json:"k8s_namespaces"`
-	S3Buckets  []healthZRespBucket `json:"s3_buckets"`
+	Namespaces       []string            `json:"k8s_namespaces"`
+	S3Buckets        []healthZRespBucket `json:"s3_buckets"`
+	SSHServerStarted bool                `json:"ssh_server_started"`
 }
 
-func healthZHandler(nsLister NamespaceLister, bLister BucketLister) http.Handler {
+func healthZHandler(nsLister NamespaceLister, bLister BucketLister, serverCircuit *sshd.Circuit) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// there's a race between the boolean eval and the HTTP error returned, but k8s will repeat the health probe request
+		// and effectively re-evaluate the boolean.
+		if serverCircuit.State() != sshd.ClosedState {
+			str := fmt.Sprintf("SSH Server is not yet started")
+			log.Err(str)
+			http.Error(w, str, http.StatusAccepted)
+			return
+		}
 		lbOut, err := bLister.ListBuckets(&s3.ListBucketsInput{})
 		if err != nil {
 			str := fmt.Sprintf("Error listing buckets (%s)", err)
@@ -53,7 +63,7 @@ func healthZHandler(nsLister NamespaceLister, bLister BucketLister) http.Handler
 		for _, ns := range nsList.Items {
 			rsp.Namespaces = append(rsp.Namespaces, ns.Name)
 		}
-
+		rsp.SSHServerStarted = true
 		if err := json.NewEncoder(w).Encode(rsp); err != nil {
 			str := fmt.Sprintf("Error encoding JSON (%s)", err)
 			http.Error(w, str, http.StatusInternalServerError)

pkg/healthsrv/server.go

@@ -3,6 +3,8 @@ package healthsrv
 import (
 	"fmt"
 	"net/http"
+
+	"github.com/deis/builder/pkg/sshd"
 )
 
 const (
@@ -11,9 +13,9 @@ const (
 )
 
 // Start starts the healthcheck server on $host:$port and blocks. It only returns if the server fails, with the indicative error
-func Start(port int, nsLister NamespaceLister, bLister BucketLister) error {
+func Start(port int, nsLister NamespaceLister, bLister BucketLister, sshServerCircuit *sshd.Circuit) error {
 	mux := http.NewServeMux()
-	mux.Handle("/healthz", healthZHandler(nsLister, bLister))
+	mux.Handle("/healthz", healthZHandler(nsLister, bLister, sshServerCircuit))
 
 	hostStr := fmt.Sprintf(":%d", port)
 	return http.ListenAndServe(hostStr, mux)

pkg/sshd/server.go

@@ -46,7 +46,7 @@ const (
 //
 // This puts the following variables into the context:
 // 	- ssh.Closer (chan interface{}): Send a message to this to shutdown the server.
-func Serve(reg *cookoo.Registry, router *cookoo.Router, c cookoo.Context) cookoo.Interrupt {
+func Serve(reg *cookoo.Registry, router *cookoo.Router, serverCircuit *Circuit, c cookoo.Context) cookoo.Interrupt {
 	hostkeys := c.Get(HostKeys, []ssh.Signer{}).([]ssh.Signer)
 	addr := c.Get(Address, "0.0.0.0:2223").(string)
 	cfg := c.Get(ServerConfig, &ssh.ServerConfig{}).(*ssh.ServerConfig)
@@ -70,6 +70,7 @@ func Serve(reg *cookoo.Registry, router *cookoo.Router, c cookoo.Context) cookoo
 	c.Put("sshd.Closer", closer)
 
 	log.Infof(c, "Listening on %s", addr)
+	serverCircuit.Close()
 	srv.listen(listener, cfg, closer)
 
 	return nil