feat(builder) : working version of decoupled builder

Author: smothiki

Makefile

@@ -2,29 +2,37 @@
 # Some uses for short name:
 # - Docker image name
 # - Kubernetes service, rc, pod, secret, volume names
-SHORT_NAME := example
+SHORT_NAME := builder
 
 # Enable vendor/ directory support.
 export GO15VENDOREXPERIMENT=1
 
 # SemVer with build information is defined in the SemVer 2 spec, but Docker
 # doesn't allow +, so we use -.
-VERSION := 0.0.1-$(shell date "+%Y%m%d%H%M%S")
+# VERSION := 0.0.1-$(shell date "+%Y%m%d%H%M%S")
 
+VERSION := 2.0.0
+BINARY_DEST_DIR := rootfs/usr/bin
 # Common flags passed into Go's linker.
 LDFLAGS := "-s -X main.version=${VERSION}"
-
+BINARIES := extract-domain extract-types extract-version generate-buildhook get-app-config get-app-values publish-release-controller yaml2json-procfile
+STANDALONE := extract-types  generate-buildhook yaml2json-procfile
 # Docker Root FS
 BINDIR := ./rootfs
 
 # Legacy support for DEV_REGISTRY, plus new support for DEIS_REGISTRY.
-DEV_REGISTRY ?= $(eval docker-machine ip deis):5000
+DEV_REGISTRY ?= $$DEV_REGISTRY
 DEIS_REGISTY ?= ${DEV_REGISTRY}
 
 # Kubernetes-specific information for RC, Service, and Image.
-RC := manifests/deis-${SHORT_NAME}-rc.yaml
-SVC := manifests/deis-${SHORT_NAME}-service.yaml
-IMAGE := ${DEIS_REGISTRY}/deis/${SHORT_NAME}:${VERSION}
+RCBP := manifests/deis-bp${SHORT_NAME}-rc.yaml
+SVCBP := manifests/deis-bp${SHORT_NAME}-service.yaml
+# IMAGEBP := ${DEV_REGISTRY}/deis/bp${SHORT_NAME}:${VERSION}
+IMAGEBP := deis/bp${SHORT_NAME}:${VERSION}
+
+RCDF := manifests/deis-df${SHORT_NAME}-rc.yaml
+SVCDF := manifests/deis-df${SHORT_NAME}-service.yaml
+IMAGEDF := ${DEV_REGISTRY}/deis/df${SHORT_NAME}:${VERSION}
 
 all:
 	@echo "Use a Makefile to control top-level building of the project."
@@ -33,34 +41,52 @@ all:
 # the Docker environment. Other alternatives are cross-compiling, doing
 # the build as a `docker build`.
 build:
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/builder cli/builder.go || exit 1
 	mkdir -p ${BINDIR}/bin
 	docker run --rm -v ${PWD}:/app -w /app golang:1.5.1 make docker-compile
 
-build-bpb:
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/builder cli/builder.go || exit 1
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/fetcher fetcher/fetcher.go || exit 1
-	@$(call check-static-binary,$(BINARY_DEST_DIR)/builder)
+docker-build-bpb:
+	cp -r bpbuilder/etcd pkg/
+	cp bpbuilder/kubectl rootfs/bin/
+	cp bpbuilder/entrypoint.sh rootfs/
+	cp bpbuilder/deis-slugbuilder.yaml rootfs/etc/
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0  go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/bpbuilder boot.go || exit 1
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0  go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/fetcher bpbuilder/fetcher/fetcher.go || exit 1
+	@$(call check-static-binary,$(BINARY_DEST_DIR)/bpbuilder)
 	@$(call check-static-binary,$(BINARY_DEST_DIR)/fetcher)
 	for i in $(BINARIES); do \
-		GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/$$i src/$$i.go || exit 1; \
+		GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/$$i pkg/src/$$i.go || exit 1; \
 	done
+	@echo "Past go compiling"
 	@for i in $(BINARIES); do \
 		$(call check-static-binary,$(BINARY_DEST_DIR)/$$i); \
 	done
-	docker build -t $(IMAGE) rootfs
+	docker build -t $(IMAGEBP) rootfs
+	perl -pi -e "s|image: [a-z0-9.:]+\/deis\/bp${SHORT_NAME}:[0-9a-z-.]+|image: ${IMAGEBP}|g" ${RCBP}
+	rm -rf pkg/etcd
+	rm rootfs/entrypoint.sh
 # For cases where build is run inside of a container.
-docker-compile:
-	go build -o ${BINDIR}/bin/boot -a -installsuffix cgo -ldflags ${LDFLAGS} boot.go
 
-# For cases where we're building from local
-# We also alter the RC file to set the image name.
-docker-build:
-	docker build --rm -t ${IMAGE} rootfs
-	perl -pi -e "s|[a-z0-9.:]+\/deis\/${SHORT_NAME}:[0-9a-z-.]+|${IMAGE}|g" ${RC}
+docker-build-dfb:
+	cp -r dfbuilder/etcd pkg/
+	cp bpbuilder/entrypoint.sh rootfs/
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/dfbuilder boot.go || exit 1
+	@$(call check-static-binary,$(BINARY_DEST_DIR)/dfbuilder)
+	for i in $(BINARIES); do \
+		GOOS=linux GOARCH=amd64 CGO_ENABLED=0 godep go build -a -installsuffix cgo -ldflags '-s' -o $(BINARY_DEST_DIR)/$$i pkg/src/$$i.go || exit 1; \
+	done
+	@for i in $(BINARIES); do \
+		$(call check-static-binary,$(BINARY_DEST_DIR)/$$i); \
+	done
+	docker build -t $(IMAGEDF) rootfs
+	perl -pi -e "s|image: [a-z0-9.:]+\/deis\/df${SHORT_NAME}:[0-9a-z-.]+|image: ${IMAGEDF}|g" ${RCDF}
+	rm -rf pkg/etcd
+	rm rootfs/entrypoint.sh
+
 
 # Push to a registry that Kubernetes can access.
-docker-push:
-	docker push ${IMAGE}
+docker-push-bp:
+	docker push ${IMAGEBP}
 
 # Deploy is a Kubernetes-oriented target
 deploy: kube-service kube-rc
@@ -78,3 +104,12 @@ kube-clean:
 	kubectl delete rc deis-example
 
 .PHONY: all build docker-compile kube-up kube-down deploy
+
+define check-static-binary
+	  if file $(1) | egrep -q "(statically linked|Mach-O)"; then \
+	    echo ""; \
+	  else \
+	    echo "The binary file $(1) is not statically linked. Build canceled"; \
+	    exit 1; \
+	  fi
+endef

_docs/README.md

@@ -1,10 +1,7 @@
-# Project Docs
+# Deis builder
 
-Documentation for the project should be stored in the `_docs` directory.
-Since the directory is protected from Go, undecorated code samples can
-also be dropped inside of this directory.
-
-We need to decide on whether we'll continue using RST or switch to
-Markdown.
+Deis builder is decoupled into two builder components.
+deis buildpack builder need deis/slugrunner and deis/slugbuilder on every host
+deis docker file builder needs deis registry running.
 
 I believe we are leaning toward MkDocs: http://www.mkdocs.org/

api/tests/builder_test.go _tests/builder_test.goapi/tests/builder_test.go renamed to _tests/builder_test.go

@@ -1,16 +1,13 @@
 package main
 
-import "fmt"
+import (
+	"os"
+	"runtime"
 
-const usage = `
-All services should provide a top-level Go program named "boot.go", compiled
-to "boot". This should go in rootfs/bin/boot, and should be the entry point
-for all Deis components.
-
-An exception to this is components that may be started without a boot, or which
-may be started with a simple (<20 line) shell script.
-`
+	"github.com/deis/builder/pkg"
+)
 
 func main() {
-	fmt.Println(usage)
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	os.Exit(pkg.Run("boot"))
 }

api/cli/README.md

@@ -64,31 +64,18 @@ TMP_DIR=$(mktemp -d -p $BUILD_DIR)
 cd $REPO_DIR
 # use Procfile if provided, otherwise try default process types from ./release
 
-
 git archive --format=tar.gz ${GIT_SHA} > ${APP_NAME}.tar.gz
 TAR_URL=http://${HOST}:3000/git/home/${SLUG_NAME}/tar
 PUSH_URL=http://${HOST}:3000/git/home/${SLUG_NAME}/push
+if [ "$nosecret" == true ]; then
+    head -n 21 /etc/slugbuilder.yaml > /etc/tmp.yaml
+    mv /etc/tmp.yaml /etc/slugbuilder.yaml
+fi
+sed -i -- 's/puturl/${PUSH_URL}/g' /etc/deis-slugbuilder.yaml
+sed -i -- 's/tar-url/${TAR_URL}/g' /etc/deis-slugbuilder.yaml
+
+kubectl create -f /etc/deis-slugbuilder.yaml
 
-docker -H ${HOST}:2375 run -e TAR_URL=${TAR_URL} smothiki/slugbuilder ${PUSH_URL}
-
-# pull config from controller to be used during build
-# if [ -f Procfile ]; then
-#     echo "here1"
-#     PROCFILE=$(cat Procfile | yaml2json-procfile)
-# elif [ -f ${APP_NAME}.tgz ]; then
-#     echo "here2"
-#     # Sometimes, the buildpack will generate a Procfile instead of populating /bin/release
-#     # /bin/release was unofficially deprecated for declaring default process types
-#     if tar -xzf ${APP_NAME}.tar.gz ./Procfile &> /dev/null;
-#     then
-#         PROCFILE="$(tar --to-stdout -xzf ${APP_NAME}.tar.gz ./Procfile | yaml2json-procfile)"
-#     else
-#         PROCFILE=$(tar --to-stdout -xzf ${APP_NAME}.tar.gz ./.release | extract-types)
-#     fi
-# else
-#     echo "here3"
-#     PROCFILE={"web":"example-go"}
-# fi
 PROCFILE='{"web":"example-go"}'
 URL="{{ getv "/deis/controller/protocol" }}://{{ getv "/deis/controller/host" }}:{{ getv "/deis/controller/port" }}/v1/hooks/config"
 RESPONSE=$(get-app-config -url="$URL" -key="{{ getv "/deis/controller/builderKey" }}" -user=$USER -app=$APP_NAME)

api/cli/builder.go

@@ -0,0 +1,29 @@
+# A debugging utility for testing Minio from within k8s.
+apiVersion: v1
+kind: Pod
+metadata:
+  name: deis-slugbuilder
+  labels:
+    heritage: deis
+    version: 2015-sept
+spec:
+  restartPolicy: Never
+  containers:
+    - name: deis-slugbuilder
+      imagePullPolicy: Always
+      image: smothiki/slugbuilder:5.3
+      env:
+        - name: DEBUG
+          value: "1"
+        - name: TAR_URL
+          value: tar-url
+        - name : put-url
+          value: puturl
+      volumeMounts:
+        - name: object-store
+          mountPath: /var/run/secrets/object/store
+          readOnly: true
+  volumes:
+    - name: object-store
+      secret:
+        secretName: object-store

boot.go

@@ -1,5 +1,5 @@
 #!/bin/bash
 set -e
 
-builder & 
+bpbuilder & 
 exec fetcher