Merge pull request #149 from arschles/probes

fix(*): add readiness & liveness probes

Author: arschles

Makefile

@@ -27,8 +27,6 @@ RC := manifests/deis-${SHORT_NAME}-rc.yaml
 SVC := manifests/deis-${SHORT_NAME}-service.yaml
 IMAGE := ${DEIS_REGISTRY}${IMAGE_PREFIX}/${SHORT_NAME}:${VERSION}
 
-TEST_PACKAGES := $(shell ${DEV_ENV_CMD} glide nv)
-
 all:
 	@echo "Use a Makefile to control top-level building of the project."
 
@@ -46,7 +44,7 @@ build:
 	@$(call check-static-binary,$(BINARY_DEST_DIR)/boot)
 
 test:
-	${DEV_ENV_CMD} go test ${TEST_PACKAGES}
+	${DEV_ENV_CMD} sh -c 'go test $$(glide nv)'
 
 docker-build:
 	docker build --rm -t ${IMAGE} rootfs

boot.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"log"
 	"os"
 	"runtime"
 
@@ -9,8 +10,11 @@ import (
 	"github.com/deis/builder/pkg"
 	"github.com/deis/builder/pkg/conf"
 	"github.com/deis/builder/pkg/gitreceive"
+	"github.com/deis/builder/pkg/gitreceive/storage"
+	"github.com/deis/builder/pkg/healthsrv"
 	"github.com/deis/builder/pkg/sshd"
 	pkglog "github.com/deis/pkg/log"
+	kcl "k8s.io/kubernetes/pkg/client/unversioned"
 )
 
 const (
@@ -26,8 +30,8 @@ func main() {
 	if os.Getenv("DEBUG") == "true" {
 		pkglog.DefaultLogger.SetDebug(true)
 		cookoolog.Level = cookoolog.LogDebug
+		log.Printf("Running in debug mode")
 	}
-	pkglog.Debug("Running in debug mode")
 
 	app := cli.NewApp()
 
@@ -42,8 +46,40 @@ func main() {
 					pkglog.Err("getting config for %s [%s]", serverConfAppName, err)
 					os.Exit(1)
 				}
-				pkglog.Info("starting SSH server on %s:%d", cnf.SSHHostIP, cnf.SSHHostPort)
-				os.Exit(pkg.Run(cnf.SSHHostIP, cnf.SSHHostPort, "boot"))
+				circ := sshd.NewCircuit()
+
+				s3Client, err := storage.GetClient(cnf.HealthSrvTestStorageRegion)
+				if err != nil {
+					log.Printf("Error getting s3 client (%s)", err)
+					os.Exit(1)
+				}
+				kubeClient, err := kcl.NewInCluster()
+				if err != nil {
+					log.Printf("Error getting kubernetes client [%s]", err)
+					os.Exit(1)
+				}
+				log.Printf("Starting health check server on port %d", cnf.HealthSrvPort)
+				healthSrvCh := make(chan error)
+				go func() {
+					if err := healthsrv.Start(cnf.HealthSrvPort, kubeClient.Namespaces(), s3Client, circ); err != nil {
+						healthSrvCh <- err
+					}
+				}()
+
+				log.Printf("Starting SSH server on %s:%d", cnf.SSHHostIP, cnf.SSHHostPort)
+				sshCh := make(chan int)
+				go func() {
+					sshCh <- pkg.RunBuilder(cnf.SSHHostIP, cnf.SSHHostPort, circ)
+				}()
+
+				select {
+				case err := <-healthSrvCh:
+					log.Printf("Error running health server (%s)", err)
+					os.Exit(1)
+				case i := <-sshCh:
+					log.Printf("Unexpected SSH server stop with code %d", i)
+					os.Exit(i)
+				}
 			},
 		},
 		{
@@ -53,13 +89,13 @@ func main() {
 			Action: func(c *cli.Context) {
 				cnf := new(gitreceive.Config)
 				if err := conf.EnvConfig(gitReceiveConfAppName, cnf); err != nil {
-					pkglog.Err("Error getting config for %s [%s]", gitReceiveConfAppName, err)
+					log.Printf("Error getting config for %s [%s]", gitReceiveConfAppName, err)
 					os.Exit(1)
 				}
 				cnf.CheckDurations()
 
 				if err := gitreceive.Run(cnf); err != nil {
-					pkglog.Err("running git receive hook [%s]", err)
+					log.Printf("Error running git receive hook [%s]", err)
 					os.Exit(1)
 				}
 			},

glide.lock

@@ -22,6 +22,7 @@ import:
   version: eca94c41d994ae2215d455ce578ae6e2dc6ee516
 - package: github.com/pborman/uuid
 - package: github.com/deis/pkg
+  version: b8679a4d7fe2fe0393c66c620eff7df86c9f7982
   subpackages:
   - time
   - log
@@ -35,3 +36,5 @@ import:
   - service/s3
 - package: k8s.io/kubernetes
   version: ~1.1
+- package: github.com/arschles/assert
+  version: 6882f85ccdc7c1822b146d1a6b0c2c48f91b5140

glide.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: ReplicationController
 metadata:
   name: deis-builder
+  namespace: deis
   labels:
     heritage: deis
 spec:
@@ -15,43 +16,50 @@ spec:
     spec:
       containers:
         - name: deis-builder
-          imagePullPolicy: Always
           image: quay.io/deisci/builder:v2-beta
+          imagePullPolicy: Always
           ports:
             - containerPort: 2223
-            - containerPort: 3000
+              name: ssh
+            - containerPort: 8092
+              name: healthsrv
           env:
-            - name: BUILDER_FETCHER_PORT
-              value: "3000"
-            - name: BUILDER_SSH_HOST_IP
-              value: "0.0.0.0"
-            - name: BUILDER_SSH_HOST_PORT
-              value: "2223"
+            - name: "HEALTH_SERVER_PORT"
+              value: "8092"
             - name: "EXTERNAL_PORT"
               value: "2223"
-            - name: POD_NAMESPACE
+            # This var needs to be passed so that the minio client (https://github.com/minio/mc) will work in Alpine linux
+            - name: "DOCKERIMAGE"
+              value: "1"
+            - name: "DEBUG"
+              value: "true"
+            - name: "POD_NAMESPACE"
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8092
+            initialDelaySeconds: 2
+            timeoutSeconds: 1
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8092
+            initialDelaySeconds: 2
+            timeoutSeconds: 1
           volumeMounts:
             - name: minio-user
               mountPath: /var/run/secrets/object/store
               readOnly: true
             - name: builder-key-auth
               mountPath: /var/run/secrets/api/auth
               readOnly: true
-            # not currently running minio with SSL support. see https://github.com/deis/minio/pull/22 for more detail
-            # - name: minio-ssl
-            #   mountPath: /var/run/secrets/object/ssl
-            #   readOnly: true
       volumes:
         - name: minio-user
           secret:
             secretName: minio-user
         - name: builder-key-auth
           secret:
             secretName: builder-key-auth
-        # not currently running minio with SSL support. see https://github.com/deis/minio/pull/22 for more detail
-        # - name: minio-ssl
-        #   secret:
-        #     secretName: minio-ssl

manifests/deis-builder-rc.yaml

@@ -2,17 +2,13 @@ apiVersion: v1
 kind: Service
 metadata:
   name: deis-builder
+  namespace: deis
   labels:
     heritage: deis
-    release: 2.0.0
 spec:
   ports:
-    - port: 2222
+    - name: ssh
+      port: 2222
       targetPort: 2223
-      name: ssh
-      protocol: TCP
-    - port: 3000
-      name: fetcher
-      protocol: TCP
   selector:
     app: deis-builder

manifests/deis-builder-service.yaml

@@ -5,13 +5,12 @@ package pkg
 
 import (
 	"fmt"
+	"log"
+	"os"
 
 	"github.com/Masterminds/cookoo"
 	clog "github.com/Masterminds/cookoo/log"
 	"github.com/deis/builder/pkg/sshd"
-
-	"log"
-	"os"
 )
 
 // Return codes that will be sent to the shell.
@@ -28,7 +27,7 @@ const (
 // Git.
 //
 // Run returns on of the Status* status code constants.
-func Run(sshHostIP string, sshHostPort int, cmd string) int {
+func RunBuilder(sshHostIP string, sshHostPort int, sshServerCircuit *sshd.Circuit) int {
 	reg, router, ocxt := cookoo.Cookoo()
 	log.SetFlags(0) // Time is captured elsewhere.
 
@@ -59,7 +58,7 @@ func Run(sshHostIP string, sshHostPort int, cmd string) int {
 	// Start the SSH service.
 	// TODO: We could refactor Serve to be a command, and then run this as
 	// a route.
-	if err := sshd.Serve(reg, router, cxt); err != nil {
+	if err := sshd.Serve(reg, router, sshServerCircuit, cxt); err != nil {
 		clog.Errf(cxt, "SSH server failed: %s", err)
 		return StatusLocalError
 	}