chore(builder): minio/minio#14331

Author: duanhongyi

README.md

@@ -35,6 +35,7 @@ Builder currently supports the following off-cluster storage backends:
 * AWS/S3
 * Azure
 * Swift
+* Alibaba OSS
 
 # Development
 

charts/builder/templates/_helpers.tpl

@@ -0,0 +1,61 @@
+{{- define "builder.envs" -}}
+env:
+# NOTE(bacongobbler): use drycc/registry_proxy to work around Docker --insecure-registry requirements
+- name: "DRYCC_REGISTRY_PROXY_HOST"
+  value: "127.0.0.1"
+- name: "DRYCC_REGISTRY_PROXY_PORT"
+  value: "{{ .Values.global.registryProxyPort }}"
+- name: "HEALTH_SERVER_PORT"
+  value: "8092"
+- name: "EXTERNAL_PORT"
+  value: "2223"
+- name: BUILDER_STORAGE
+  value: "{{ .Values.global.storage }}"
+- name: "DRYCC_REGISTRY_LOCATION"
+  value: "{{ .Values.global.registryLocation }}"
+- name: "TTL_SECONDS_AFTER_FINISHED"
+  value: "{{ .Values.global.ttlSecondsAfterFinished }}"
+# Set GIT_LOCK_TIMEOUT to number of minutes you want to wait to git push again to the same repository
+- name: "GIT_LOCK_TIMEOUT"
+  value: "30"
+- name: IMAGEBUILDER_IMAGE_PULL_POLICY
+  valueFrom:
+    configMapKeyRef:
+      name: imagebuilder-config
+      key: imagePullPolicy
+- name: "DRYCC_DEBUG"
+  value: "false"
+- name: "POD_NAMESPACE"
+  valueFrom:
+    fieldRef:
+    fieldPath: metadata.namespace
+- name: DRYCC_BUILDER_KEY
+  valueFrom:
+    secretKeyRef:
+      name: builder-key-auth
+      key: builder-key
+{{- if (.Values.builder_pod_node_selector) }}
+- name: BUILDER_POD_NODE_SELECTOR
+  value: {{.Values.builder_pod_node_selector}}
+{{- if eq .Values.global.minioLocation "on-cluster" }}
+- name: "DRYCC_MINIO_ENDPOINT"
+  value: ${DRYCC_MINIO_SERVICE_HOST}:${DRYCC_MINIO_SERVICE_PORT}
+{{- else }}
+- name: "DRYCC_MINIO_ENDPOINT"
+  value: "{{ .Values.minio.endpoint }}"
+{{- end }}
+{{- end }}
+
+{{/* Generate builder deployment limits */}}
+{{- define "builder.limits" -}}
+{{- if or (.Values.limitsCpu) (.Values.limitsMemory) }}
+resources:
+  limits:
+    {{- if (.Values.limitsCpu) }}
+    cpu: {{.Values.limitsCpu}}
+    {{- end }}
+    {{- if (.Values.limitsMemory) }}
+    memory: {{.Values.limitsMemory}}
+    {{- end }}
+{{- end }}
+{{- end }}

charts/builder/templates/builder-deployment.yaml

@@ -63,8 +63,8 @@ spec:
             - name: builder-ssh-private-keys
               mountPath: /var/run/secrets/drycc/builder/ssh
               readOnly: true
-            - name: objectstore-creds
-              mountPath: /var/run/secrets/drycc/objectstore/creds
+            - name: minio-creds
+              mountPath: /var/run/secrets/drycc/minio/creds
               readOnly: true
             - name: imagebuilder-config
               mountPath: /etc/imagebuilder
@@ -76,9 +76,9 @@ spec:
         - name: builder-ssh-private-keys
           secret:
             secretName: builder-ssh-private-keys
-        - name: objectstore-creds
+        - name: minio-creds
           secret:
-            secretName: objectstorage-keyfile
+            secretName: minio-creds
         - name: imagebuilder-config
           configMap:
             name: imagebuilder-config

pkg/cleaner/cleaner.go

@@ -88,7 +88,7 @@ func dirHasGitSuffix(dir string) bool {
 	return strings.HasSuffix(dir, dotGitSuffix)
 }
 
-func deleteFromObjectStore(app string, storageDriver storagedriver.StorageDriver) error {
+func deleteFromMinio(app string, storageDriver storagedriver.StorageDriver) error {
 
 	// delete all files matching app
 	objs, err := storageDriver.List(context.Background(), "home")
@@ -138,7 +138,7 @@ func Run(gitHome string, nsLister k8s.NamespaceLister, fs sys.FS, pollSleepDurat
 			if err := fs.RemoveAll(dirToDelete); err != nil {
 				log.Err("Cleaner error removing local files for deleted app %s (%s)", dirToDelete, err)
 			}
-			if err := deleteFromObjectStore(appToDelete, storageDriver); err != nil {
+			if err := deleteFromMinio(appToDelete, storageDriver); err != nil {
 				log.Err("Cleaner error removing object store files for deleted app %s (%s)", appToDelete, err)
 			}
 		}

pkg/conf/config.go

@@ -9,9 +9,8 @@ import (
 )
 
 const (
-	storageCredLocation = "/var/run/secrets/drycc/objectstore/creds/"
-	minioHostEnvVar     = "DRYCC_MINIO_ENDPOINT"
-	gcsKey              = "key.json"
+	storageCredLocation = "/var/run/secrets/drycc/minio/creds/"
+	minioEndpointVar    = "DRYCC_MINIO_ENDPOINT"
 )
 
 // BuilderKeyLocation holds the path of the builder key secret.
@@ -33,6 +32,7 @@ func GetBuilderKey() (string, error) {
 // GetStorageParams returns the credentials required for connecting to object storage
 func GetStorageParams(env sys.Env) (Parameters, error) {
 	params := make(map[string]interface{})
+	params["builder-bucket"] = "builder" // default
 	files, err := ioutil.ReadDir(storageCredLocation)
 	if err != nil {
 		return nil, err
@@ -46,18 +46,13 @@ func GetStorageParams(env sys.Env) (Parameters, error) {
 		if err != nil {
 			return nil, err
 		}
-		//GCS expect the to have the location of the service account credential json file
-		if file.Name() == gcsKey {
-			params["keyfile"] = storageCredLocation + file.Name()
-		} else {
-			params[file.Name()] = string(data)
-		}
+
+		params[file.Name()] = string(data)
 	}
 	params["bucket"] = params["builder-bucket"]
-	mHost := env.Get(minioHostEnvVar)
-	mPort := env.Get(minioPortEnvVar)
+	mEndpointVar := env.Get(minioEndpointVar)
 	params["region"] = "us-east-1"
-	params["regionendpoint"] = fmt.Sprintf("http://%s:%s", mHost, mPort)
+	params["regionendpoint"] = fmt.Sprintf("http://%s", mEndpointVar)
 	params["secure"] = false
 	return params, nil
 }

pkg/conf/config_test.go

@@ -78,7 +78,7 @@ func TestGetStorageParams(t *testing.T) {
 	assert.Equal(t, params["regionendpoint"], "http://localhost:8088", "region endpoint")
 	assert.Equal(t, params["secure"], false, "secure")
 	assert.Equal(t, params["region"], "us-east-1", "region")
-	assert.Equal(t, params["bucket"], nil, "bucket")
+	assert.Equal(t, params["bucket"], "builder", "bucket")
 }
 
 func TestGetControllerClient(t *testing.T) {

pkg/gitreceive/k8s_util.go

@@ -23,9 +23,9 @@ const (
 	tarPath                = "TAR_PATH"
 	debugKey               = "DRYCC_DEBUG"
 	sourceVersion          = "SOURCE_VERSION"
-	objectStore            = "objectstorage-keyfile"
+	minioCreds             = "minio-creds"
 	builderStorage         = "BUILDER_STORAGE"
-	objectStorePath        = "/var/run/secrets/drycc/objectstore/creds"
+	minioCredsPath         = "/var/run/secrets/drycc/minio/creds"
 	imagebuilderConfig     = "imagebuilder-config"
 	imagebuilderConfigPath = "/etc/imagebuilder"
 )
@@ -144,18 +144,18 @@ func buildJob(
 	job.Spec.Template.Spec.Containers[0].ImagePullPolicy = pullPolicy
 	job.Spec.Template.Spec.Containers[0].SecurityContext = &securityContext
 	job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
-		Name: objectStore,
+		Name: minioCreds,
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
-				SecretName: objectStore,
+				SecretName: minioCreds,
 			},
 		},
 	})
 
 	job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
 		{
-			Name:      objectStore,
-			MountPath: objectStorePath,
+			Name:      minioCreds,
+			MountPath: minioCredsPath,
 			ReadOnly:  true,
 		},
 	}

rootfs/bin/normalize_storage

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
-BUCKET_FILE="/var/run/secrets/drycc/objectstore/creds/builder-bucket"
-ACCESS_KEY_FILE="/var/run/secrets/drycc/objectstore/creds/accesskey"
-SECRET_KEY_FILE="/var/run/secrets/drycc/objectstore/creds/secretkey"
+BUCKET_FILE="/var/run/secrets/drycc/minio/creds/builderBucket"
+ACCESS_KEY_FILE="/var/run/secrets/drycc/minio/creds/accesskey"
+SECRET_KEY_FILE="/var/run/secrets/drycc/minio/creds/secretkey"
 
 if [ -f $BUCKET_FILE ]; then
   MINIO_BUCKET=$(cat "$BUCKET_FILE")