ref(registry):Enable registry to use multiple object stores

Author: Keerthan Mala

Dockerfile

@@ -21,5 +21,4 @@ COPY rootfs/ /
 # define the execution environment
 VOLUME ["/var/lib/registry"]
 EXPOSE 5000
-ENTRYPOINT ["/bin/registry"]
-CMD ["/etc/docker/registry/config.yml"]
+CMD ["/opt/registry/sbin/registry"]

Makefile

@@ -8,36 +8,60 @@ SHORT_NAME := registry
 
 VERSION ?= git-$(shell git rev-parse --short HEAD)
 
+# the filepath to this repository, relative to $GOPATH/src
+REPO_PATH = github.com/deis/registry
+
+# The following variables describe the containerized development environment
+# and other build options
+DEV_ENV_IMAGE := quay.io/deis/go-dev:0.7.0
+DEV_ENV_WORK_DIR := /go/src/${REPO_PATH}
+DEV_ENV_CMD := docker run --rm -v ${CURDIR}:${DEV_ENV_WORK_DIR} -w ${DEV_ENV_WORK_DIR} ${DEV_ENV_IMAGE}
+LDFLAGS := "-s -X main.version=${VERSION}"
+BINDIR := ./rootfs/opt/registry/sbin
+
 # Legacy support for DEV_REGISTRY, plus new support for DEIS_REGISTRY.
 DEIS_REGISTRY ?= ${DEV_REGISTRY}
 
 IMAGE_PREFIX ?= deis
 
-# Kubernetes-specific information for RC, Service, and Image.
+
+ifeq ($(STORAGE_TYPE),)
+  STORAGE_TYPE = fs
+endif
+
+# Kubernetes-specific information for Secret, RC, Service, and Image.
+SECRET := contrib/kubernetes/manifests/${SHORT_NAME}-${STORAGE_TYPE}-secret.yaml
 RC := contrib/kubernetes/manifests/${SHORT_NAME}-rc.yaml
 SVC := contrib/kubernetes/manifests/${SHORT_NAME}-service.yaml
 IMAGE := ${DEIS_REGISTRY}${IMAGE_PREFIX}/${SHORT_NAME}:${VERSION}
 
 all:
 	@echo "Use a Makefile to control top-level building of the project."
 
-build:
-	@echo "Nothing to build. Use 'make docker-build' to build the image."
+build: check-docker
+	mkdir -p ${BINDIR}
+	${DEV_ENV_CMD} make build-binary
 
 # For cases where we're building from local
 # We also alter the RC file to set the image name.
-docker-build: check-docker
+docker-build: check-docker build
 	docker build --rm -t ${IMAGE} .
 
 # Push to a registry that Kubernetes can access.
 docker-push: check-docker check-registry
 	docker push ${IMAGE}
 
+build-binary:
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags ${LDFLAGS} -o $(BINDIR)/${SHORT_NAME} main.go
+
 # Deploy is a Kubernetes-oriented target
-deploy: kube-service kube-rc
+deploy: kube-secret kube-service kube-rc
+
+kube-secret: check-kubectl
+	kubectl create -f ${SECRET}
 
 # Some things, like services, have to be deployed before pods. This is an
-# example target. Others could perhaps include kube-secret, kube-volume, etc.
+# example target. Others could perhaps include kube-volume, etc.
 kube-service: check-kubectl
 	kubectl create -f ${SVC}
 

README.md

@@ -44,6 +44,20 @@ Before deploying your custom image you must update the container image specifica
               value: "true"
 ```
 
+By default registry uses the filesystem as the storage medium. To use a custom object store like s3 or gcs:
+- First provide the details required for authenticating to object store in base64 format by updating the secret file which can be found at `contrib/kubernetes/manifests/registry-{STORAGE_TYPE}-secret.yaml`.
+- Set the STORAGE_TYPE environment variable.
+```
+$ export STORAGE_TYPE = {s3/gcs}
+```
+- Update the secret to be used in the pod manifest. This file is found at `contrib/kubernetes/manifests/registry-rc.yaml`:
+```yaml
+        - name: registry-creds
+          secret:
+            secretName: fs-keyfile
+```
+
+
 Once updated, deploy the registry to your kubernetes cluster with:
 
 ```

contrib/kubernetes/manifests/registry-fs-secret.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: fs-keyfile
+type: Opaque
+data:

contrib/kubernetes/manifests/registry-gcs-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: gcs-keyfile
+type: Opaque
+data:
+  key.json: {base64-encoded service account JSON file data}
+  bucket: {base64-encoded bucket name}

contrib/kubernetes/manifests/registry-rc.yaml

@@ -19,11 +19,18 @@ spec:
               value: "true"
             - name: REGISTRY_LOG_LEVEL
               value: info
+            - name: REGISTRY_STORAGE
+              value: filesystem
           ports:
             - containerPort: 5000
           volumeMounts:
             - name: registry-storage
               mountPath: /var/lib/registry
+            - name: registry-creds
+              mountPath: /var/run/secrets/deis/registry/creds
       volumes:
         - name: registry-storage
           emptyDir: {}
+        - name: registry-creds
+          secret:
+            secretName: fs-keyfile

contrib/kubernetes/manifests/registry-s3-secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: s3-keyfile
+type: Opaque
+data:
+  accesskey: {base64-encoded access key}
+  secretkey: {base64-encoded secret key}
+  region: {base64-encoded region name}
+  bucket: {base64-encoded bucket name}

main.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+)
+
+const (
+	registryBinary = "/bin/registry"
+	registryConfig = "/etc/docker/registry/config.yml"
+)
+
+func main() {
+	log.Println("INFO: Starting registry...")
+	storageType := getenv("REGISTRY_STORAGE", "filesystem")
+	if storageType == "gcs" {
+		if _, err := os.Stat("/var/run/secrets/deis/registry/creds/key.json"); err != nil {
+			log.Fatal("Service account not given")
+		}
+		os.Setenv("REGISTRY_STORAGE_GCS_KEYFILE", "/var/run/secrets/deis/registry/creds/key.json")
+		if bucket, err := ioutil.ReadFile("/var/run/secrets/deis/registry/creds/bucket"); err != nil {
+			log.Fatal(err)
+		} else {
+			os.Setenv("REGISTRY_STORAGE_GCS_BUCKET", string(bucket))
+		}
+	} else if storageType == "s3" {
+		if accesskey, err := ioutil.ReadFile("/var/run/secrets/deis/registry/creds/accesskey"); err != nil {
+			log.Fatal(err)
+		} else {
+			os.Setenv("REGISTRY_STORAGE_S3_ACCESSKEY", string(accesskey))
+		}
+
+		if secretkey, err := ioutil.ReadFile("/var/run/secrets/deis/registry/creds/secretkey"); err != nil {
+			log.Fatal(err)
+		} else {
+			os.Setenv("REGISTRY_STORAGE_S3_SECRETKEY", string(secretkey))
+		}
+
+		if region, err := ioutil.ReadFile("/var/run/secrets/deis/registry/creds/region"); err != nil {
+			log.Fatal(err)
+		} else {
+			os.Setenv("REGISTRY_STORAGE_S3_REGION", string(region))
+		}
+
+		if bucket, err := ioutil.ReadFile("/var/run/secrets/deis/registry/creds/bucket"); err != nil {
+			log.Fatal(err)
+		} else {
+			os.Setenv("REGISTRY_STORAGE_S3_BUCKET", string(bucket))
+		}
+	}
+
+	cmd := exec.Command(registryBinary, registryConfig)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		log.Fatal("Error starting the registry", err)
+	}
+	log.Println("INFO: registry started.")
+	for {
+	}
+}
+
+func getenv(name, dfault string) string {
+	value := os.Getenv(name)
+	if value == "" {
+		value = dfault
+	}
+	return value
+}