Merge pull request #11 from jianxiaoguo/main

chore(builder): run imagebuider replace pod with job

Author: duanhongyi

.gitignore

@@ -4,3 +4,5 @@ rootfs/usr/bin/
 ./builder
 coverage.txt
 testdata/hooks/pre-receive
+.idea/
+

charts/builder/templates/builder-deployment.yaml

@@ -64,6 +64,8 @@ spec:
             value: "{{ .Values.global.storage }}"
           - name: "DRYCC_REGISTRY_LOCATION"
             value: "{{ .Values.global.registry_location }}"
+          - name: "TTL_SECONDS_AFTER_FINISHED"
+            value: "{{ .Values.global.ttl_seconds_after_finished }}"
           # Set GIT_LOCK_TIMEOUT to number of minutes you want to wait to git push again to the same repository
           - name: "GIT_LOCK_TIMEOUT"
             value: "30"

charts/builder/templates/builder-role.yaml

@@ -16,4 +16,7 @@ rules:
 - apiGroups: [""]
   resources: ["pods/log"]
   verbs: ["get"]
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["create"]
 {{- end -}}

charts/builder/values.yaml

@@ -11,6 +11,10 @@ service:
 # limits_memory: "50Mi"
 # builder_pod_node_selector: "disk:ssd"
 
+# When the TTL controller cleans up the Job. default: 6h
+# see: https://kubernetes.io/docs/concepts/workloads/controllers/job/#ttl-mechanism-for-finished-jobs
+ttl_seconds_after_finished: 21600
+
 global:
   # Role-Based Access Control for Kubernetes >= 1.5
   use_rbac: false

pkg/gitreceive/build.go

@@ -148,7 +148,7 @@ func build(
 	securityContext := k8s.SecurityContextFromPrivileged(true)
 
 	imageName := fmt.Sprintf("%s:git-%s", appName, gitSha.Short())
-	buildPodName := imagebuilderPodName(appName, gitSha.Short())
+	buildJobName := imagebuilderJobName(appName, gitSha.Short())
 	registryLocation := conf.RegistryLocation
 	builderImageEnv := make(map[string]string)
 	if registryLocation != "on-cluster" {
@@ -160,9 +160,9 @@ func build(
 	builderImageEnv["DRYCC_STACK"] = stack["name"]
 	builderImageEnv["DRYCC_REGISTRY_LOCATION"] = registryLocation
 
-	pod := createBuilderPod(
+	job := createBuilderJob(
 		conf.Debug,
-		buildPodName,
+		buildJobName,
 		conf.PodNamespace,
 		appConf.Values,
 		tarKey,
@@ -181,17 +181,16 @@ func build(
 
 	log.Info("Starting build... but first, coffee!")
 	log.Debug("Use image %s: %s", stack["name"], stack["image"])
-	log.Debug("Starting pod %s", buildPodName)
-	json, err := prettyPrintJSON(pod)
+	log.Debug("Starting job %s", buildJobName)
+	json, err := prettyPrintJSON(job)
 	if err == nil {
-		log.Debug("Pod spec: %v", json)
+		log.Debug("Job spec: %v", json)
 	} else {
-		log.Debug("Error creating json representation of pod spec: %v", err)
+		log.Debug("Error creating json representation of Job spec: %v", err)
 	}
+	jobsInterface := kubeClient.BatchV1().Jobs(conf.PodNamespace)
 
-	podsInterface := kubeClient.CoreV1().Pods(conf.PodNamespace)
-
-	newPod, err := podsInterface.Create(ctx.TODO(), pod, metav1.CreateOptions{})
+	newJob, err := jobsInterface.Create(ctx.TODO(), job, metav1.CreateOptions{})
 	if err != nil {
 		return fmt.Errorf("creating builder pod (%s)", err)
 	}
@@ -201,11 +200,19 @@ func build(
 	defer close(stopCh)
 	go pw.Controller.Run(stopCh)
 
-	if err := waitForPod(pw, newPod.Namespace, newPod.Name, conf.SessionIdleInterval(), conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
+	if err := waitForPod(pw, newJob.Namespace, newJob.Name, conf.SessionIdleInterval(), conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
 		return fmt.Errorf("watching events for builder pod startup (%s)", err)
 	}
 
-	req := kubeClient.CoreV1().RESTClient().Get().Namespace(newPod.Namespace).Name(newPod.Name).Resource("pods").SubResource("log").VersionedParams(
+	options := metav1.ListOptions{
+		LabelSelector: fmt.Sprintf("heritage=%s", newJob.Name),
+	}
+	podList, err := kubeClient.CoreV1().Pods(newJob.Namespace).List(context.Background(), options)
+	if err != nil {
+		return fmt.Errorf("list pods %s fail: (%s)", newJob.Name, err)
+	}
+
+	req := kubeClient.CoreV1().RESTClient().Get().Namespace(newJob.Namespace).Name(podList.Items[0].Name).Resource("pods").SubResource("log").VersionedParams(
 		&corev1.PodLogOptions{
 			Follow: true,
 		}, scheme.ParameterCodec)
@@ -224,19 +231,19 @@ func build(
 
 	log.Debug(
 		"Waiting for the %s/%s pod to end. Checking every %s for %s",
-		newPod.Namespace,
-		newPod.Name,
+		newJob.Namespace,
+		newJob.Name,
 		conf.BuilderPodTickDuration(),
 		conf.BuilderPodWaitDuration(),
 	)
 	// check the state and exit code of the build pod.
 	// if the code is not 0 return error
-	if err := waitForPodEnd(pw, newPod.Namespace, newPod.Name, conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
+	if err := waitForPodEnd(pw, newJob.Namespace, newJob.Name, conf.BuilderPodTickDuration(), conf.BuilderPodWaitDuration()); err != nil {
 		return fmt.Errorf("error getting builder pod status (%s)", err)
 	}
 	log.Debug("Done")
 	log.Debug("Checking for builder pod exit code")
-	buildPod, err := kubeClient.CoreV1().Pods(newPod.Namespace).Get(ctx.TODO(), newPod.Name, metav1.GetOptions{})
+	buildPod, err := kubeClient.CoreV1().Pods(newJob.Namespace).Get(ctx.TODO(), podList.Items[0].Name, metav1.GetOptions{})
 	if err != nil {
 		return fmt.Errorf("error getting builder pod status (%s)", err)
 	}

pkg/gitreceive/k8s_util.go

@@ -4,10 +4,13 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"os"
+	"strconv"
 	"time"
 
 	"github.com/drycc/builder/pkg/k8s"
 	"github.com/pborman/uuid"
+	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,7 +30,7 @@ const (
 	imagebuilderConfigPath = "/etc/imagebuilder"
 )
 
-func imagebuilderPodName(appName, shortSha string) string {
+func imagebuilderJobName(appName, shortSha string) string {
 	uid := uuid.New()[:8]
 	// NOTE(bacongobbler): pod names cannot exceed 63 characters in length, so we truncate
 	// the application name to stay under that limit when adding all the extra metadata to the name
@@ -37,7 +40,7 @@ func imagebuilderPodName(appName, shortSha string) string {
 	return fmt.Sprintf("imagebuild-%s-%s-%s", appName, shortSha, uid)
 }
 
-func createBuilderPod(
+func createBuilderJob(
 	debug bool,
 	name,
 	namespace string,
@@ -54,9 +57,9 @@ func createBuilderPod(
 	pullPolicy corev1.PullPolicy,
 	securityContext corev1.SecurityContext,
 	nodeSelector map[string]string,
-) *corev1.Pod {
+) *batchv1.Job {
 
-	pod := buildPod(debug, name, namespace, pullPolicy, securityContext, nodeSelector, env)
+	job := buildJob(debug, name, namespace, pullPolicy, securityContext, nodeSelector, env)
 
 	// inject application envvars as a special envvar which will be handled by imagebuilder to
 	// inject them as build-time variables.
@@ -67,47 +70,67 @@ func createBuilderPod(
 	// So we need to translate the map into json.
 	if _, ok := env["DRYCC_DOCKER_BUILD_ARGS_ENABLED"]; ok {
 		imageBuildArgs, _ := json.Marshal(env)
-		addEnvToPod(pod, "DOCKER_BUILD_ARGS", string(imageBuildArgs))
+		addEnvToJob(job, "DOCKER_BUILD_ARGS", string(imageBuildArgs))
 	}
+	job.Spec.Template.Spec.Containers[0].Name = builderName
+	job.Spec.Template.Spec.Containers[0].Image = builderImage
 
-	pod.Spec.Containers[0].Name = builderName
-	pod.Spec.Containers[0].Image = builderImage
-
-	addEnvToPod(pod, tarPath, tarKey)
-	addEnvToPod(pod, sourceVersion, gitShortHash)
-	addEnvToPod(pod, "IMG_NAME", imageName)
-	addEnvToPod(pod, builderStorage, storageType)
+	addEnvToJob(job, tarPath, tarKey)
+	addEnvToJob(job, sourceVersion, gitShortHash)
+	addEnvToJob(job, "IMG_NAME", imageName)
+	addEnvToJob(job, builderStorage, storageType)
 	// inject existing DRYCC_REGISTRY_PROXY_HOST and PORT info to imagebuilder
 	// see https://github.com/drycc/imagebuilder/issues/83
-	addEnvToPod(pod, "DRYCC_REGISTRY_PROXY_HOST", registryHost)
-	addEnvToPod(pod, "DRYCC_REGISTRY_PROXY_PORT", registryPort)
+	addEnvToJob(job, "DRYCC_REGISTRY_PROXY_HOST", registryHost)
+	addEnvToJob(job, "DRYCC_REGISTRY_PROXY_PORT", registryPort)
 
 	for key, value := range builderImageEnv {
-		addEnvToPod(pod, key, value)
+		addEnvToJob(job, key, value)
 	}
 
-	return &pod
+	return &job
+}
+
+func newInt32(i int32) *int32 {
+	return &i
 }
 
-func buildPod(
+func buildJob(
 	debug bool,
 	name,
 	namespace string,
 	//pullPolicy api.PullPolicy,
 	pullPolicy corev1.PullPolicy,
 	securityContext corev1.SecurityContext,
 	nodeSelector map[string]string,
-	env map[string]interface{}) corev1.Pod {
-	pod := corev1.Pod{
-		Spec: corev1.PodSpec{
-			RestartPolicy: corev1.RestartPolicyNever,
-			Containers: []corev1.Container{
-				{
-					ImagePullPolicy: pullPolicy,
-					SecurityContext: &securityContext,
+	env map[string]interface{}) batchv1.Job {
+	TTLSecondsAfterFinished := newInt32(21600)
+	if os.Getenv("TTL_SECONDS_AFTER_FINISHED") != "" {
+		ttl, err := strconv.ParseInt(os.Getenv("TTL_SECONDS_AFTER_FINISHED"), 10, 32)
+		if err == nil {
+			TTLSecondsAfterFinished = newInt32(int32(ttl))
+		}
+	}
+
+	job := batchv1.Job{
+		Spec: batchv1.JobSpec{
+			BackoffLimit:            newInt32(0),
+			TTLSecondsAfterFinished: TTLSecondsAfterFinished,
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{
+						"heritage": name,
+					},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							ImagePullPolicy: pullPolicy,
+							SecurityContext: &securityContext,
+						},
+					},
 				},
 			},
-			Volumes: []corev1.Volume{},
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
@@ -117,8 +140,10 @@ func buildPod(
 			},
 		},
 	}
-
-	pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+	job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever
+	job.Spec.Template.Spec.Containers[0].ImagePullPolicy = pullPolicy
+	job.Spec.Template.Spec.Containers[0].SecurityContext = &securityContext
+	job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
 		Name: objectStore,
 		VolumeSource: corev1.VolumeSource{
 			Secret: &corev1.SecretVolumeSource{
@@ -127,14 +152,14 @@ func buildPod(
 		},
 	})
 
-	pod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
+	job.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{
 		{
 			Name:      objectStore,
 			MountPath: objectStorePath,
 			ReadOnly:  true,
 		},
 	}
-	pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{
+	job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
 		Name: imagebuilderConfig,
 		VolumeSource: corev1.VolumeSource{
 			ConfigMap: &corev1.ConfigMapVolumeSource{
@@ -145,35 +170,35 @@ func buildPod(
 		},
 	})
 
-	pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
+	job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
 		Name:      imagebuilderConfig,
 		MountPath: imagebuilderConfigPath,
 		ReadOnly:  true,
 	})
 
-	if len(pod.Spec.Containers) > 0 {
+	if len(job.Spec.Template.Spec.Containers) > 0 {
 		for k, v := range env {
-			pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{
+			job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
 				Name:  k,
 				Value: fmt.Sprintf("%v", v),
 			})
 		}
 	}
 
 	if len(nodeSelector) > 0 {
-		pod.Spec.NodeSelector = nodeSelector
+		job.Spec.Template.Spec.NodeSelector = nodeSelector
 	}
 
 	if debug {
-		addEnvToPod(pod, debugKey, "1")
+		addEnvToJob(job, debugKey, "1")
 	}
 
-	return pod
+	return job
 }
 
-func addEnvToPod(pod corev1.Pod, key, value string) {
-	if len(pod.Spec.Containers) > 0 {
-		pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, corev1.EnvVar{
+func addEnvToJob(job batchv1.Job, key, value string) {
+	if len(job.Spec.Template.Spec.Containers) > 0 {
+		job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
 			Name:  key,
 			Value: value,
 		})